{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 引入相关的包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from collections import Counter\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "from tensorflow.python.ops import math_ops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据的处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 评分数据\n",
    "分别有用户ID、电影ID、评分和时间戳等字段，数据中的格式：UserID::MovieID::Rating::Timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Rating</th>\n",
       "      <th>timestamps</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserId  MovieID  Rating  timestamps\n",
       "0       1     1193       5   978300760\n",
       "1       1      661       3   978302109\n",
       "2       1      914       3   978301968\n",
       "3       1     3408       4   978300275\n",
       "4       1     2355       5   978824291"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_title = ['UserId' , 'MovieID' , 'Rating' , 'timestamps']\n",
    "ratings = pd.read_table('../data/ratings.dat' , sep='::' , header=None , names=rating_title , engine='python')\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用户数据\n",
    "分别有用户ID、性别、年龄、职业ID和邮编等字段。 数据中的格式：UserID::Gender::Age::Occupation::Zip-code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>OccupationID</th>\n",
       "      <th>Zip-code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>48067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>M</td>\n",
       "      <td>56</td>\n",
       "      <td>16</td>\n",
       "      <td>70072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>55117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>M</td>\n",
       "      <td>45</td>\n",
       "      <td>7</td>\n",
       "      <td>02460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>20</td>\n",
       "      <td>55455</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserID Gender  Age  OccupationID Zip-code\n",
       "0       1      F    1            10    48067\n",
       "1       2      M   56            16    70072\n",
       "2       3      M   25            15    55117\n",
       "3       4      M   45             7    02460\n",
       "4       5      M   25            20    55455"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_title = ['UserID' , 'Gender' , 'Age' , 'OccupationID' , 'Zip-code']\n",
    "users = pd.read_table('../data/users.dat' , sep = '::' , header=None , names=users_title , engine='python')\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 电影数据\n",
    "分别有电影ID、电影名和电影风格等字段。数据中的格式：MovieID::Title::Genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>Heat (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>Sabrina (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Tom and Huck (1995)</td>\n",
       "      <td>Adventure|Children's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Sudden Death (1995)</td>\n",
       "      <td>Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>Action|Adventure|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>American President, The (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>Dracula: Dead and Loving It (1995)</td>\n",
       "      <td>Comedy|Horror</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>Balto (1995)</td>\n",
       "      <td>Animation|Children's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>Nixon (1995)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>Cutthroat Island (1995)</td>\n",
       "      <td>Action|Adventure|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>Casino (1995)</td>\n",
       "      <td>Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>Sense and Sensibility (1995)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>Ace Ventura: When Nature Calls (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>Money Train (1995)</td>\n",
       "      <td>Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>Action|Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>Assassins (1995)</td>\n",
       "      <td>Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>Powder (1995)</td>\n",
       "      <td>Drama|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>Leaving Las Vegas (1995)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>Othello (1995)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>Now and Then (1995)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>Persuasion (1995)</td>\n",
       "      <td>Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>City of Lost Children, The (1995)</td>\n",
       "      <td>Adventure|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>Shanghai Triad (Yao a yao yao dao waipo qiao) ...</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    MovieID                                              Title  \\\n",
       "0         1                                   Toy Story (1995)   \n",
       "1         2                                     Jumanji (1995)   \n",
       "2         3                            Grumpier Old Men (1995)   \n",
       "3         4                           Waiting to Exhale (1995)   \n",
       "4         5                 Father of the Bride Part II (1995)   \n",
       "5         6                                        Heat (1995)   \n",
       "6         7                                     Sabrina (1995)   \n",
       "7         8                                Tom and Huck (1995)   \n",
       "8         9                                Sudden Death (1995)   \n",
       "9        10                                   GoldenEye (1995)   \n",
       "10       11                     American President, The (1995)   \n",
       "11       12                 Dracula: Dead and Loving It (1995)   \n",
       "12       13                                       Balto (1995)   \n",
       "13       14                                       Nixon (1995)   \n",
       "14       15                            Cutthroat Island (1995)   \n",
       "15       16                                      Casino (1995)   \n",
       "16       17                       Sense and Sensibility (1995)   \n",
       "17       18                                  Four Rooms (1995)   \n",
       "18       19              Ace Ventura: When Nature Calls (1995)   \n",
       "19       20                                 Money Train (1995)   \n",
       "20       21                                  Get Shorty (1995)   \n",
       "21       22                                     Copycat (1995)   \n",
       "22       23                                   Assassins (1995)   \n",
       "23       24                                      Powder (1995)   \n",
       "24       25                           Leaving Las Vegas (1995)   \n",
       "25       26                                     Othello (1995)   \n",
       "26       27                                Now and Then (1995)   \n",
       "27       28                                  Persuasion (1995)   \n",
       "28       29                  City of Lost Children, The (1995)   \n",
       "29       30  Shanghai Triad (Yao a yao yao dao waipo qiao) ...   \n",
       "\n",
       "                          Genres  \n",
       "0    Animation|Children's|Comedy  \n",
       "1   Adventure|Children's|Fantasy  \n",
       "2                 Comedy|Romance  \n",
       "3                   Comedy|Drama  \n",
       "4                         Comedy  \n",
       "5          Action|Crime|Thriller  \n",
       "6                 Comedy|Romance  \n",
       "7           Adventure|Children's  \n",
       "8                         Action  \n",
       "9      Action|Adventure|Thriller  \n",
       "10          Comedy|Drama|Romance  \n",
       "11                 Comedy|Horror  \n",
       "12          Animation|Children's  \n",
       "13                         Drama  \n",
       "14      Action|Adventure|Romance  \n",
       "15                Drama|Thriller  \n",
       "16                 Drama|Romance  \n",
       "17                      Thriller  \n",
       "18                        Comedy  \n",
       "19                        Action  \n",
       "20           Action|Comedy|Drama  \n",
       "21          Crime|Drama|Thriller  \n",
       "22                      Thriller  \n",
       "23                  Drama|Sci-Fi  \n",
       "24                 Drama|Romance  \n",
       "25                         Drama  \n",
       "26                         Drama  \n",
       "27                       Romance  \n",
       "28              Adventure|Sci-Fi  \n",
       "29                         Drama  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_title = ['MovieID' , 'Title' , 'Genres']\n",
    "movies = pd.read_table('../data/movies.dat' , sep='::' , header=None , names=movie_title , engine='python')\n",
    "movies.head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据处理的部分\n",
    "\n",
    "UserID、Occupation和MovieID不用变。\n",
    "\n",
    "Gender字段：需要将'F'和'M'转换成0和1。\n",
    "\n",
    "Age字段：要转成7个连续数字0~6,初始数据就是表示范围的。\n",
    "\n",
    "Genres字段：是分类字段，要转成数字。首先将Genres中的类别转成字符串到数字的字典，然后再将每个电影的Genres字段\n",
    "转成数字列表，因为有些电影是多个Genres的组合。\n",
    "\n",
    "Title字段：处理方式跟Genres字段一样，首先创建文本到数字的字典，然后将Title中的描述转成数字的列表。另外Title中的年份也需要去掉。\n",
    "\n",
    "Genres和Title字段需要将长度统一，这样在神经网络中方便处理。空白部分用‘< PAD >’对应的数字填充。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取数据，并且对数据进行预处理\n",
    "def load_data():\n",
    "    # 读取用户数据\n",
    "    users_title = ['UserID' , 'Gender' , 'Age' , 'JobID' , 'Zip-code']\n",
    "    users = pd.read_table(\"../data/users.dat\",sep = '::',header=None,names=users_title,engine='python')\n",
    "    users = users.filter(regex = 'UserID|Gender|Age|JobID')  # 按照列进行过滤，只保留这 4 列\n",
    "    users_origin = users.values  # 把 users的值保存这个副本\n",
    "    \n",
    "    # 对用户性别进行处理\n",
    "    gender_map = {'F':0 , 'M':1}\n",
    "    users['Gender'] = users['Gender'].map(gender_map)  # 对每个元素进行 map映射\n",
    "    \n",
    "    # 对用户年龄进行处理\n",
    "    age_map = {val:i for i,val in enumerate(set(users['Age']))} # 之所以用 set是因为去除重复\n",
    "    users['Age'] = users['Age'].map(age_map)   # Age的value 部分是 0-6\n",
    "    \n",
    "    # 读取电影数据\n",
    "    movie_title = ['MovieID' , 'Title' , 'Genres']\n",
    "    movies = pd.read_table('../data/movies.dat',sep='::',header=None,names=movie_title,engine='python')\n",
    "    movies_origin = movies.values  # 把 movies的值保存这个副本\n",
    "    \n",
    "    # 把 Title中的年份去掉\n",
    "    pattern = re.compile(r'^(.*)\\((\\d+)\\)$')\n",
    "    title_map = {val:pattern.match(val).group(1) for i,val in enumerate(set(movies['Title']))}\n",
    "    movies['Title'] = movies['Title'].map(title_map)\n",
    "    \n",
    "    # 电影类型转为数字的set\n",
    "    genres_set = set()\n",
    "    for val in movies['Genres'].str.split('|'):\n",
    "        genres_set.update(val) # 修改当前集合，把val包含的元素全部加入到 genres_set中且去除重复\n",
    "    \n",
    "    # 长度统一，空白部分用 '<PAD>'填充\n",
    "    genres_set.add('<PAD>')\n",
    "    # 将电影类型的 set改为 map  value部分是电影类型所对应的 int\n",
    "    genres2int = {val:i for i , val in enumerate(genres_set)}\n",
    "    \n",
    "    #将电影类型转成等长数字列表，长度是\n",
    "    # 下面得到的是一个 map, key 是 电影类型（可以组合），value是个List，表示电影类型的 数字组合\n",
    "    genres_map = {val:[genres2int[row] for row in val.split('|')] for i,val in enumerate(set(movies['Genres']))}\n",
    "\n",
    "    \n",
    "    #将电影类型转成等长数字列表，长度是18，因为一共18种电影\n",
    "    for key in genres_map:\n",
    "        for cnt in range(max(genres2int.values()) - len(genres_map[key])):\n",
    "            genres_map[key].insert(len(genres_map[key]) + cnt,genres2int['<PAD>'])\n",
    "    \n",
    "    movies['Genres'] = movies['Genres'].map(genres_map)\n",
    "    \n",
    "    # # 生成一个set，里面包含了每个Title中每个词\n",
    "    title_set = set()\n",
    "    for val in movies['Title'].str.split():   # 没有参数，按照空格换行符制表符等分割\n",
    "        title_set.update(val)\n",
    "    title_set.add('<PAD>')\n",
    "    \n",
    "    # 然后把每个词转化为 int类型\n",
    "    title2int = {val:i for i , val in enumerate(title_set)}\n",
    "    \n",
    "    #将电影 Title转为等长的数字列表，长度为15 \n",
    "    title_count = 15\n",
    "    # key是电影Title，value 是电影Title分割出来的词的int 表示\n",
    "    title_map = {val:[title2int[row] for row in val.split()] for i,val in enumerate(set(movies['Title']))}\n",
    "    \n",
    "    for key in title_map:\n",
    "        for cnt in range(title_count - len(title_map[key])):\n",
    "            title_map[key].insert(len(title_map[key]) + cnt,title2int['<PAD>'])\n",
    "    # 这样就把原先 movies['Title']改为了 List[int]表示\n",
    "    movies['Title'] = movies['Title'].map(title_map) \n",
    "    \n",
    "    \n",
    "    # 读取评分数据集\n",
    "    ratings_title = ['UserID' , 'MovieID' , 'ratings' , 'timestamps']\n",
    "    ratings = pd.read_table('../data/ratings.dat', sep='::', header=None, names=ratings_title, engine = 'python')\n",
    "    ratings = ratings.filter(regex='UserID|MovieID|ratings')\n",
    "    \n",
    "    # 合并三个表\n",
    "    data = pd.merge(pd.merge(ratings , users) , movies)\n",
    "    \n",
    "    #将数据分成X和y两张表\n",
    "    target_fields = ['ratings']\n",
    "    features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields]\n",
    "    \n",
    "    features = features_pd.values\n",
    "    targets_values = targets_pd.values\n",
    "    \n",
    "    return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_origin, users_origin    \n",
    "\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 加载数据，并且保存到本地\n",
    "\n",
    "title_count：Title字段的长度（15）\n",
    "title_set：Title文本的集合，保存的是各个电影Title拆分后的词\n",
    "genres2int：电影类型转数字的字典，保存的也是词\n",
    "features：是输入X,是除了 ratings的其他部分的数据\n",
    "targets_values：是学习目标y，也就是 ratings的数据\n",
    "ratings：评分数据集的Pandas对象\n",
    "users：用户数据集的Pandas对象\n",
    "movies：电影数据的Pandas对象\n",
    "data：三个数据集组合在一起的Pandas对象\n",
    "movies_orig：没有做数据处理的原始电影数据\n",
    "users_orig：没有做数据处理的原始用户数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 序列化对象，并将结果数据流写入到文件对象中\n",
    "title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig = load_data()\n",
    "pickle.dump((title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig), open('../../Movie_Recommender_data/preprocess.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>JobID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    UserID  Gender  Age  JobID\n",
       "0        1       0    0     10\n",
       "1        2       1    5     16\n",
       "2        3       1    6     15\n",
       "3        4       1    2      7\n",
       "4        5       1    6     20\n",
       "5        6       0    3      9\n",
       "6        7       1    1      1\n",
       "7        8       1    6     12\n",
       "8        9       1    6     17\n",
       "9       10       0    1      1\n",
       "10      11       0    6      1\n",
       "11      12       1    6     12\n",
       "12      13       1    2      1\n",
       "13      14       1    1      0\n",
       "14      15       1    6      7\n",
       "15      16       0    1      0\n",
       "16      17       1    3      1\n",
       "17      18       0    4      3\n",
       "18      19       1    0     10\n",
       "19      20       1    6     14\n",
       "20      21       1    4     16\n",
       "21      22       1    4     15\n",
       "22      23       1    1      0\n",
       "23      24       0    6      7\n",
       "24      25       1    4      4\n",
       "25      26       1    6      7\n",
       "26      27       1    6     11\n",
       "27      28       0    6      1\n",
       "28      29       1    1      7\n",
       "29      30       0    1      7"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>[1083, 4396, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[17, 5, 16, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>[2663, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[0, 5, 11, 15, 15, 15, 15, 15, 15, 15, 15, 15,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[374, 2184, 1593, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[16, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>[2308, 4315, 960, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[16, 18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>[4967, 1664, 633, 3294, 117, 4631, 4969, 4969,...</td>\n",
       "      <td>[16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>[4605, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[1, 12, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>[354, 4969, 4969, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[16, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>[1560, 218, 3723, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>[4432, 699, 4969, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[1, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>[5176, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[1, 0, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>[1999, 2053, 627, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[16, 18, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>[1823, 806, 218, 4866, 2760, 4969, 4969, 4969,...</td>\n",
       "      <td>[16, 13, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>[791, 4969, 4969, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[17, 5, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>[703, 4969, 4969, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>[3517, 1736, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[1, 0, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>[1269, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[18, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>[4864, 218, 4167, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[18, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>[3277, 4707, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>[3292, 48, 1488, 2020, 2607, 4969, 4969, 4969,...</td>\n",
       "      <td>[16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>[132, 2780, 4969, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[1, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>[3163, 3651, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[1, 16, 18, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>[2861, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[12, 18, 14, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>[3050, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>[1002, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[18, 6, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>[3146, 1466, 1552, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[18, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>[1454, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>[3276, 218, 1582, 4969, 4969, 4969, 4969, 4969...</td>\n",
       "      <td>[18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>[3942, 4969, 4969, 4969, 4969, 4969, 4969, 496...</td>\n",
       "      <td>[8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>[1355, 1664, 2848, 1264, 627, 4969, 4969, 4969...</td>\n",
       "      <td>[0, 6, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>[3537, 4428, 4394, 3824, 1976, 1976, 1026, 200...</td>\n",
       "      <td>[18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    MovieID                                              Title  \\\n",
       "0         1  [1083, 4396, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "1         2  [2663, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "2         3  [374, 2184, 1593, 4969, 4969, 4969, 4969, 4969...   \n",
       "3         4  [2308, 4315, 960, 4969, 4969, 4969, 4969, 4969...   \n",
       "4         5  [4967, 1664, 633, 3294, 117, 4631, 4969, 4969,...   \n",
       "5         6  [4605, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "6         7  [354, 4969, 4969, 4969, 4969, 4969, 4969, 4969...   \n",
       "7         8  [1560, 218, 3723, 4969, 4969, 4969, 4969, 4969...   \n",
       "8         9  [4432, 699, 4969, 4969, 4969, 4969, 4969, 4969...   \n",
       "9        10  [5176, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "10       11  [1999, 2053, 627, 4969, 4969, 4969, 4969, 4969...   \n",
       "11       12  [1823, 806, 218, 4866, 2760, 4969, 4969, 4969,...   \n",
       "12       13  [791, 4969, 4969, 4969, 4969, 4969, 4969, 4969...   \n",
       "13       14  [703, 4969, 4969, 4969, 4969, 4969, 4969, 4969...   \n",
       "14       15  [3517, 1736, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "15       16  [1269, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "16       17  [4864, 218, 4167, 4969, 4969, 4969, 4969, 4969...   \n",
       "17       18  [3277, 4707, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "18       19  [3292, 48, 1488, 2020, 2607, 4969, 4969, 4969,...   \n",
       "19       20  [132, 2780, 4969, 4969, 4969, 4969, 4969, 4969...   \n",
       "20       21  [3163, 3651, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "21       22  [2861, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "22       23  [3050, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "23       24  [1002, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "24       25  [3146, 1466, 1552, 4969, 4969, 4969, 4969, 496...   \n",
       "25       26  [1454, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "26       27  [3276, 218, 1582, 4969, 4969, 4969, 4969, 4969...   \n",
       "27       28  [3942, 4969, 4969, 4969, 4969, 4969, 4969, 496...   \n",
       "28       29  [1355, 1664, 2848, 1264, 627, 4969, 4969, 4969...   \n",
       "29       30  [3537, 4428, 4394, 3824, 1976, 1976, 1026, 200...   \n",
       "\n",
       "                                               Genres  \n",
       "0   [17, 5, 16, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "1   [0, 5, 11, 15, 15, 15, 15, 15, 15, 15, 15, 15,...  \n",
       "2   [16, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "3   [16, 18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "4   [16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "5   [1, 12, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "6   [16, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "7   [0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,...  \n",
       "8   [1, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "9   [1, 0, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15,...  \n",
       "10  [16, 18, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "11  [16, 13, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "12  [17, 5, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "13  [18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "14  [1, 0, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, ...  \n",
       "15  [18, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "16  [18, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "17  [14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "18  [16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "19  [1, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "20  [1, 16, 18, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "21  [12, 18, 14, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "22  [14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "23  [18, 6, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "24  [18, 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "25  [18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "26  [18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  \n",
       "27  [8, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15...  \n",
       "28  [0, 6, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,...  \n",
       "29  [18, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 1...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1,\n",
       "       list([1083, 4396, 4969, 4969, 4969, 4969, 4969, 4969, 4969, 4969, 4969, 4969, 4969, 4969, 4969]),\n",
       "       list([17, 5, 16, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15])],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 从本地读取进行了特征工程后的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig = pickle.load(open('../../Movie_Recommender_data/preprocess.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 网络的搭建"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 辅助函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os \n",
    "import pickle\n",
    "\n",
    "def save_params(params):\n",
    "    \"\"\"\n",
    "    Save parameters to file\n",
    "    \"\"\"\n",
    "    pickle.dump(params, open('params.p', 'wb'))\n",
    "\n",
    "\n",
    "def load_params():\n",
    "    \"\"\"\n",
    "    Load parameters from file\n",
    "    \"\"\"\n",
    "    return pickle.load(open('params.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 编码实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 嵌入矩阵的维度\n",
    "embed_dim = 32\n",
    "# 用户ID个数,take()是ndarray的方法，第1个参数的列，第二个参数是axis，表示得到列中对应的各个行组成的 ndarray\n",
    "uid_max = max(features.take(0,1)) + 1 # 6040\n",
    "# 性别个数\n",
    "gender_max = max(features.take(2,1)) + 1 # 1 + 1 = 2\n",
    "#年龄类别个数\n",
    "age_max = max(features.take(3,1)) + 1 # 6 + 1 = 7\n",
    "#职业个数\n",
    "job_max = max(features.take(4,1)) + 1# 20 + 1 = 21\n",
    "\n",
    "#电影ID个数\n",
    "movie_id_max = max(features.take(1,1)) + 1 # 3952\n",
    "#电影类型个数\n",
    "movie_categories_max = max(genres2int.values()) + 1 # 18 + 1 = 19\n",
    "#电影名单词个数\n",
    "movie_title_max = len(title_set) # 5216\n",
    "\n",
    "#对电影类型嵌入向量做加和操作的标志，考虑过使用mean做平均，但是没实现mean\n",
    "combiner = \"sum\"\n",
    "\n",
    "#电影名长度\n",
    "sentences_size = title_count # = 15\n",
    "#文本卷积滑动窗口，分别滑动2, 3, 4, 5个单词\n",
    "window_sizes = {2, 3, 4, 5}\n",
    "#文本卷积核数量\n",
    "filter_num = 8\n",
    "\n",
    "#电影ID转下标的字典，数据集中电影ID跟下标不一致，比如第5行的数据电影ID不一定是5\n",
    "movieid2idx = {val[0]:i for i, val in enumerate(movies.values)}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Epochs\n",
    "num_epochs = 5\n",
    "# Batch Size\n",
    "batch_size = 256\n",
    "\n",
    "dropout_keep = 0.5\n",
    "# Learning Rate\n",
    "learning_rate = 0.0001\n",
    "# Show stats for every n number of batches\n",
    "show_every_n_batches = 20\n",
    "\n",
    "save_dir = './save'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义输入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_inputs():\n",
    "    uid = tf.placeholder(tf.int32, [None, 1], name=\"uid\")\n",
    "    user_gender = tf.placeholder(tf.int32, [None, 1], name=\"user_gender\")\n",
    "    user_age = tf.placeholder(tf.int32, [None, 1], name=\"user_age\")\n",
    "    user_job = tf.placeholder(tf.int32, [None, 1], name=\"user_job\")\n",
    "    \n",
    "    movie_id = tf.placeholder(tf.int32, [None, 1], name=\"movie_id\")\n",
    "    movie_categories = tf.placeholder(tf.int32, [None, 18], name=\"movie_categories\")\n",
    "    movie_titles = tf.placeholder(tf.int32, [None, 15], name=\"movie_titles\")\n",
    "    targets = tf.placeholder(tf.int32, [None, 1], name=\"targets\")\n",
    "    LearningRate = tf.placeholder(tf.float32, name = \"LearningRate\")\n",
    "    dropout_keep_prob = tf.placeholder(tf.float32, name = \"dropout_keep_prob\")\n",
    "    return uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, LearningRate, dropout_keep_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建神经网络"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义User的嵌入矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_embedding(uid, user_gender, user_age, user_job):\n",
    "    with tf.name_scope(\"user_embedding\"):   # 经过这个就少了很多列了\n",
    "        # 下面两句是先取均匀的随机数，然后根据 uid取相应的列\n",
    "        uid_embed_matrix = tf.Variable(tf.random_uniform([uid_max, embed_dim], -1, 1), name = \"uid_embed_matrix\")\n",
    "        uid_embed_layer = tf.nn.embedding_lookup(uid_embed_matrix, uid, name = \"uid_embed_layer\")\n",
    "        \n",
    "        # 下面三个的列数目都只有 16 而不是 32  然后行数 是各不相同的\n",
    "        gender_embed_matrix = tf.Variable(tf.random_uniform([gender_max, embed_dim // 2], -1, 1), name= \"gender_embed_matrix\")\n",
    "        gender_embed_layer = tf.nn.embedding_lookup(gender_embed_matrix, user_gender, name = \"gender_embed_layer\")\n",
    "        \n",
    "        age_embed_matrix = tf.Variable(tf.random_uniform([age_max, embed_dim // 2], -1, 1), name=\"age_embed_matrix\")\n",
    "        age_embed_layer = tf.nn.embedding_lookup(age_embed_matrix, user_age, name=\"age_embed_layer\")\n",
    "        \n",
    "        job_embed_matrix = tf.Variable(tf.random_uniform([job_max, embed_dim // 2], -1, 1), name = \"job_embed_matrix\")\n",
    "        job_embed_layer = tf.nn.embedding_lookup(job_embed_matrix, user_job, name = \"job_embed_layer\")\n",
    "    return uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将User的嵌入矩阵一起全连接生成 User 的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_feature_layer(uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer):\n",
    "    with tf.name_scope(\"user_fc\"):\n",
    "        #第一层全连接，输出维度（也就是这层的节点数目）为 embed_dim\n",
    "        uid_fc_layer = tf.layers.dense(uid_embed_layer, embed_dim, name = \"uid_fc_layer\", activation=tf.nn.relu)\n",
    "        gender_fc_layer = tf.layers.dense(gender_embed_layer, embed_dim, name = \"gender_fc_layer\", activation=tf.nn.relu)\n",
    "        age_fc_layer = tf.layers.dense(age_embed_layer, embed_dim, name =\"age_fc_layer\", activation=tf.nn.relu)\n",
    "        job_fc_layer = tf.layers.dense(job_embed_layer, embed_dim, name = \"job_fc_layer\", activation=tf.nn.relu)\n",
    "        \n",
    "        #第二层全连接,所以这里是 4*32=128\n",
    "        user_combine_layer = tf.concat([uid_fc_layer, gender_fc_layer, age_fc_layer, job_fc_layer], 2)  #(?, 1, 128)\n",
    "        user_combine_layer = tf.contrib.layers.fully_connected(user_combine_layer, 200, tf.tanh)  #(?, 1, 200)\n",
    "    \n",
    "        user_combine_layer_flat = tf.reshape(user_combine_layer, [-1, 200])\n",
    "    return user_combine_layer, user_combine_layer_flat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义Movie ID的嵌入矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_movie_id_embed_layer(movie_id):\n",
    "    with tf.name_scope(\"movie_embedding\"):\n",
    "        movie_id_embed_matrix = tf.Variable(tf.random_uniform([movie_id_max, embed_dim], -1, 1), name = \"movie_id_embed_matrix\")\n",
    "        movie_id_embed_layer = tf.nn.embedding_lookup(movie_id_embed_matrix, movie_id, name = \"movie_id_embed_layer\")\n",
    "    return movie_id_embed_layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对电影类型的多个嵌入向量做加和"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_categories_layers(movie_categories):\n",
    "    with tf.name_scope(\"movie_categories_layers\"):\n",
    "        movie_categories_embed_matrix = tf.Variable(tf.random_uniform([movie_categories_max, embed_dim], -1, 1), name = \"movie_categories_embed_matrix\")\n",
    "        movie_categories_embed_layer = tf.nn.embedding_lookup(movie_categories_embed_matrix, movie_categories, name = \"movie_categories_embed_layer\")\n",
    "        if combiner == \"sum\":\n",
    "            # axis=1就是按行求和，有几行结果就有几个元素，可以理解为数据压缩\n",
    "            movie_categories_embed_layer = tf.reduce_sum(movie_categories_embed_layer, axis=1, keep_dims=True)\n",
    "    #     elif combiner == \"mean\":\n",
    "\n",
    "    return movie_categories_embed_layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Movie Title的文本卷积网络实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_cnn_layer(movie_titles):\n",
    "    #从嵌入矩阵中得到电影名对应的各个单词的嵌入向量\n",
    "    with tf.name_scope(\"movie_embedding\"):\n",
    "        movie_title_embed_matrix = tf.Variable(tf.random_uniform([movie_title_max, embed_dim], -1, 1), name = \"movie_title_embed_matrix\")\n",
    "        movie_title_embed_layer = tf.nn.embedding_lookup(movie_title_embed_matrix, movie_titles, name = \"movie_title_embed_layer\")\n",
    "        # 组成4维的输入，对于图片，是 图片数量, 图片高度, 图片宽度, 图像通道数\n",
    "        movie_title_embed_layer_expand = tf.expand_dims(movie_title_embed_layer, -1)\n",
    "    \n",
    "    #对文本嵌入层使用不同尺寸的卷积核做卷积和最大池化\n",
    "    pool_layer_lst = []\n",
    "    for window_size in window_sizes:\n",
    "        with tf.name_scope(\"movie_txt_conv_maxpool_{}\".format(window_size)):\n",
    "            # 卷积层是4维，对于图片，是 卷积核的高度，卷积核的宽度，图像通道数，卷积核个数\n",
    "            filter_weights = tf.Variable(tf.truncated_normal([window_size, embed_dim, 1, filter_num],stddev=0.1),name = \"filter_weights\")\n",
    "            filter_bias = tf.Variable(tf.constant(0.1, shape=[filter_num]), name=\"filter_bias\")\n",
    "            \n",
    "            # 卷积层，输入4维、卷积核4维、四个维度的步长[1,1,1,1]\n",
    "            conv_layer = tf.nn.conv2d(movie_title_embed_layer_expand, filter_weights, [1,1,1,1], padding=\"VALID\", name=\"conv_layer\")\n",
    "            # 对于每个元素，将大于0的数保持不变，小于0的数置为0\n",
    "            # 一般 relu层都是跟在卷积操作+偏置 的后面\n",
    "            relu_layer = tf.nn.relu(tf.nn.bias_add(conv_layer,filter_bias), name =\"relu_layer\")\n",
    "            # 最大池化\n",
    "            maxpool_layer = tf.nn.max_pool(relu_layer, [1,sentences_size - window_size + 1 ,1,1], [1,1,1,1], padding=\"VALID\", name=\"maxpool_layer\")\n",
    "            # 把卷积层+relu层+池化层 一起组合添加到 list中\n",
    "            pool_layer_lst.append(maxpool_layer)\n",
    "\n",
    "    #Dropout层\n",
    "    with tf.name_scope(\"pool_dropout\"):\n",
    "        # 回忆文本卷积网络，把各个层连接起来,组合池化后的全部特征并拉平\n",
    "        pool_layer = tf.concat(pool_layer_lst, 3, name =\"pool_layer\")\n",
    "        max_num = len(window_sizes) * filter_num\n",
    "        pool_layer_flat = tf.reshape(pool_layer , [-1, 1, max_num], name = \"pool_layer_flat\")\n",
    "        # dropout 防止过拟合 \n",
    "        dropout_layer = tf.nn.dropout(pool_layer_flat, dropout_keep_prob, name = \"dropout_layer\")\n",
    "    return pool_layer_flat, dropout_layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将Movie的各个层一起做全连接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_feature_layer(movie_id_embed_layer, movie_categories_embed_layer, dropout_layer):\n",
    "    with tf.name_scope(\"movie_fc\"):\n",
    "        #电影ID 和 电影类型 的全连接层\n",
    "        movie_id_fc_layer = tf.layers.dense(movie_id_embed_layer, embed_dim, name = \"movie_id_fc_layer\", activation=tf.nn.relu)\n",
    "        movie_categories_fc_layer = tf.layers.dense(movie_categories_embed_layer, embed_dim, name = \"movie_categories_fc_layer\", activation=tf.nn.relu)\n",
    "    \n",
    "        # 电影ID、电影类型和 电影名进行dropout后的文本卷积层 连接  \n",
    "        movie_combine_layer = tf.concat([movie_id_fc_layer, movie_categories_fc_layer, dropout_layer], 2)  #(?, 1, 96)\n",
    "        movie_combine_layer = tf.contrib.layers.fully_connected(movie_combine_layer, 200, tf.tanh)  #(?, 1, 200)\n",
    "        \n",
    "        #铺平 \n",
    "        movie_combine_layer_flat = tf.reshape(movie_combine_layer, [-1, 200])\n",
    "    return movie_combine_layer, movie_combine_layer_flat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建计算图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-18-8f2260751882>:7: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    #获取输入占位符\n",
    "    uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob = get_inputs()\n",
    "    #获取User的4个嵌入向量\n",
    "    uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer = get_user_embedding(uid, user_gender, user_age, user_job)\n",
    "    #得到用户特征\n",
    "    user_combine_layer, user_combine_layer_flat = get_user_feature_layer(uid_embed_layer, gender_embed_layer, age_embed_layer, job_embed_layer)\n",
    "    #获取电影ID的嵌入向量\n",
    "    movie_id_embed_layer = get_movie_id_embed_layer(movie_id)\n",
    "    #获取电影类型的嵌入向量\n",
    "    movie_categories_embed_layer = get_movie_categories_layers(movie_categories)\n",
    "    #获取电影名的特征向量\n",
    "    pool_layer_flat, dropout_layer = get_movie_cnn_layer(movie_titles)\n",
    "    #得到电影特征\n",
    "    movie_combine_layer, movie_combine_layer_flat = get_movie_feature_layer(movie_id_embed_layer, \n",
    "                                                                                movie_categories_embed_layer, \n",
    "                                                                                dropout_layer)\n",
    "    #计算出评分，要注意两个不同的方案，inference的名字（name值）是不一样的，后面做推荐时要根据name取得tensor\n",
    "    with tf.name_scope(\"inference\"):\n",
    "        #将用户特征和电影特征作为输入，经过全连接，输出一个值的方案\n",
    "        #简单的将用户特征和电影特征做矩阵乘法得到一个预测评分\n",
    "        inference = tf.reduce_sum(user_combine_layer_flat * movie_combine_layer_flat, axis=1)\n",
    "        inference = tf.expand_dims(inference, axis=1)\n",
    "\n",
    "    with tf.name_scope(\"loss\"):\n",
    "        # MSE损失，将计算值回归到评分\n",
    "        cost = tf.losses.mean_squared_error(targets, inference )\n",
    "        loss = tf.reduce_mean(cost)\n",
    "    # 优化损失 \n",
    "    global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "    optimizer = tf.train.AdamOptimizer(lr)\n",
    "    gradients = optimizer.compute_gradients(loss)  #cost\n",
    "    train_op = optimizer.apply_gradients(gradients, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'inference/ExpandDims:0' shape=(?, 1) dtype=float32>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 取得batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batches(Xs, ys, batch_size):\n",
    "    for start in range(0, len(Xs), batch_size):\n",
    "        end = min(start + batch_size, len(Xs))\n",
    "        yield Xs[start:end], ys[start:end]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing to E:\\jupyter-src\\Movie_Recommender\\code\\runs\\1553068050\n",
      "\n",
      "2019-03-20T15:47:32.056449: Epoch   0 Batch    0/3125   train_loss = 10.119\n",
      "2019-03-20T15:47:32.617920: Epoch   0 Batch   20/3125   train_loss = 4.451\n",
      "2019-03-20T15:47:33.246350: Epoch   0 Batch   40/3125   train_loss = 3.337\n",
      "2019-03-20T15:47:33.892144: Epoch   0 Batch   60/3125   train_loss = 2.335\n",
      "2019-03-20T15:47:34.531483: Epoch   0 Batch   80/3125   train_loss = 2.096\n",
      "2019-03-20T15:47:35.182234: Epoch   0 Batch  100/3125   train_loss = 1.975\n",
      "2019-03-20T15:47:35.836952: Epoch   0 Batch  120/3125   train_loss = 1.848\n",
      "2019-03-20T15:47:36.466375: Epoch   0 Batch  140/3125   train_loss = 1.804\n",
      "2019-03-20T15:47:37.105224: Epoch   0 Batch  160/3125   train_loss = 1.419\n",
      "2019-03-20T15:47:37.747044: Epoch   0 Batch  180/3125   train_loss = 1.669\n",
      "2019-03-20T15:47:38.382914: Epoch   0 Batch  200/3125   train_loss = 1.645\n",
      "2019-03-20T15:47:39.015809: Epoch   0 Batch  220/3125   train_loss = 1.564\n",
      "2019-03-20T15:47:39.647215: Epoch   0 Batch  240/3125   train_loss = 1.470\n",
      "2019-03-20T15:47:40.273166: Epoch   0 Batch  260/3125   train_loss = 1.522\n",
      "2019-03-20T15:47:40.897133: Epoch   0 Batch  280/3125   train_loss = 1.726\n",
      "2019-03-20T15:47:41.532011: Epoch   0 Batch  300/3125   train_loss = 1.417\n",
      "2019-03-20T15:47:42.180282: Epoch   0 Batch  320/3125   train_loss = 1.459\n",
      "2019-03-20T15:47:42.804745: Epoch   0 Batch  340/3125   train_loss = 1.277\n",
      "2019-03-20T15:47:43.449542: Epoch   0 Batch  360/3125   train_loss = 1.497\n",
      "2019-03-20T15:47:44.085416: Epoch   0 Batch  380/3125   train_loss = 1.373\n",
      "2019-03-20T15:47:44.711364: Epoch   0 Batch  400/3125   train_loss = 1.200\n",
      "2019-03-20T15:47:45.411219: Epoch   0 Batch  420/3125   train_loss = 1.426\n",
      "2019-03-20T15:47:46.090242: Epoch   0 Batch  440/3125   train_loss = 1.482\n",
      "2019-03-20T15:47:46.740495: Epoch   0 Batch  460/3125   train_loss = 1.510\n",
      "2019-03-20T15:47:47.417534: Epoch   0 Batch  480/3125   train_loss = 1.350\n",
      "2019-03-20T15:47:48.096557: Epoch   0 Batch  500/3125   train_loss = 1.156\n",
      "2019-03-20T15:47:48.769627: Epoch   0 Batch  520/3125   train_loss = 1.331\n",
      "2019-03-20T15:47:49.413931: Epoch   0 Batch  540/3125   train_loss = 1.267\n",
      "2019-03-20T15:47:50.104855: Epoch   0 Batch  560/3125   train_loss = 1.388\n",
      "2019-03-20T15:47:50.741717: Epoch   0 Batch  580/3125   train_loss = 1.329\n",
      "2019-03-20T15:47:51.404868: Epoch   0 Batch  600/3125   train_loss = 1.423\n",
      "2019-03-20T15:47:52.075459: Epoch   0 Batch  620/3125   train_loss = 1.324\n",
      "2019-03-20T15:47:52.737617: Epoch   0 Batch  640/3125   train_loss = 1.425\n",
      "2019-03-20T15:47:53.405729: Epoch   0 Batch  660/3125   train_loss = 1.317\n",
      "2019-03-20T15:47:54.092190: Epoch   0 Batch  680/3125   train_loss = 1.170\n",
      "2019-03-20T15:47:54.753357: Epoch   0 Batch  700/3125   train_loss = 1.349\n",
      "2019-03-20T15:47:55.428909: Epoch   0 Batch  720/3125   train_loss = 1.215\n",
      "2019-03-20T15:47:56.111402: Epoch   0 Batch  740/3125   train_loss = 1.427\n",
      "2019-03-20T15:47:56.774056: Epoch   0 Batch  760/3125   train_loss = 1.329\n",
      "2019-03-20T15:47:57.424807: Epoch   0 Batch  780/3125   train_loss = 1.411\n",
      "2019-03-20T15:47:58.075061: Epoch   0 Batch  800/3125   train_loss = 1.263\n",
      "2019-03-20T15:47:58.707459: Epoch   0 Batch  820/3125   train_loss = 1.268\n",
      "2019-03-20T15:47:59.352754: Epoch   0 Batch  840/3125   train_loss = 1.231\n",
      "2019-03-20T15:47:59.972753: Epoch   0 Batch  860/3125   train_loss = 1.131\n",
      "2019-03-20T15:48:00.585311: Epoch   0 Batch  880/3125   train_loss = 1.232\n",
      "2019-03-20T15:48:01.207295: Epoch   0 Batch  900/3125   train_loss = 1.237\n",
      "2019-03-20T15:48:01.832749: Epoch   0 Batch  920/3125   train_loss = 1.217\n",
      "2019-03-20T15:48:02.451259: Epoch   0 Batch  940/3125   train_loss = 1.403\n",
      "2019-03-20T15:48:03.079193: Epoch   0 Batch  960/3125   train_loss = 1.364\n",
      "2019-03-20T15:48:03.691260: Epoch   0 Batch  980/3125   train_loss = 1.373\n",
      "2019-03-20T15:48:04.305303: Epoch   0 Batch 1000/3125   train_loss = 1.263\n",
      "2019-03-20T15:48:04.915877: Epoch   0 Batch 1020/3125   train_loss = 1.354\n",
      "2019-03-20T15:48:05.544307: Epoch   0 Batch 1040/3125   train_loss = 1.256\n",
      "2019-03-20T15:48:06.180674: Epoch   0 Batch 1060/3125   train_loss = 1.445\n",
      "2019-03-20T15:48:06.823490: Epoch   0 Batch 1080/3125   train_loss = 1.249\n",
      "2019-03-20T15:48:07.429599: Epoch   0 Batch 1100/3125   train_loss = 1.347\n",
      "2019-03-20T15:48:08.040670: Epoch   0 Batch 1120/3125   train_loss = 1.275\n",
      "2019-03-20T15:48:08.619004: Epoch   0 Batch 1140/3125   train_loss = 1.207\n",
      "2019-03-20T15:48:09.215196: Epoch   0 Batch 1160/3125   train_loss = 1.264\n",
      "2019-03-20T15:48:09.790554: Epoch   0 Batch 1180/3125   train_loss = 1.205\n",
      "2019-03-20T15:48:10.375832: Epoch   0 Batch 1200/3125   train_loss = 1.229\n",
      "2019-03-20T15:48:10.970535: Epoch   0 Batch 1220/3125   train_loss = 1.101\n",
      "2019-03-20T15:48:11.544902: Epoch   0 Batch 1240/3125   train_loss = 1.126\n",
      "2019-03-20T15:48:12.179284: Epoch   0 Batch 1260/3125   train_loss = 1.250\n",
      "2019-03-20T15:48:12.768531: Epoch   0 Batch 1280/3125   train_loss = 1.221\n",
      "2019-03-20T15:48:13.366706: Epoch   0 Batch 1300/3125   train_loss = 1.143\n",
      "2019-03-20T15:48:13.948016: Epoch   0 Batch 1320/3125   train_loss = 1.189\n",
      "2019-03-20T15:48:14.557599: Epoch   0 Batch 1340/3125   train_loss = 1.058\n",
      "2019-03-20T15:48:15.157261: Epoch   0 Batch 1360/3125   train_loss = 1.203\n",
      "2019-03-20T15:48:15.746508: Epoch   0 Batch 1380/3125   train_loss = 1.049\n",
      "2019-03-20T15:48:16.387836: Epoch   0 Batch 1400/3125   train_loss = 1.267\n",
      "2019-03-20T15:48:17.035113: Epoch   0 Batch 1420/3125   train_loss = 1.292\n",
      "2019-03-20T15:48:17.637255: Epoch   0 Batch 1440/3125   train_loss = 1.195\n",
      "2019-03-20T15:48:18.208646: Epoch   0 Batch 1460/3125   train_loss = 1.239\n",
      "2019-03-20T15:48:18.792439: Epoch   0 Batch 1480/3125   train_loss = 1.188\n",
      "2019-03-20T15:48:19.373252: Epoch   0 Batch 1500/3125   train_loss = 1.343\n",
      "2019-03-20T15:48:20.016066: Epoch   0 Batch 1520/3125   train_loss = 1.258\n",
      "2019-03-20T15:48:20.612258: Epoch   0 Batch 1540/3125   train_loss = 1.298\n",
      "2019-03-20T15:48:21.288304: Epoch   0 Batch 1560/3125   train_loss = 1.105\n",
      "2019-03-20T15:48:21.985182: Epoch   0 Batch 1580/3125   train_loss = 1.247\n",
      "2019-03-20T15:48:22.634940: Epoch   0 Batch 1600/3125   train_loss = 1.226\n",
      "2019-03-20T15:48:23.334298: Epoch   0 Batch 1620/3125   train_loss = 1.188\n",
      "2019-03-20T15:48:23.945369: Epoch   0 Batch 1640/3125   train_loss = 1.297\n",
      "2019-03-20T15:48:24.573799: Epoch   0 Batch 1660/3125   train_loss = 1.247\n",
      "2019-03-20T15:48:25.299446: Epoch   0 Batch 1680/3125   train_loss = 1.195\n",
      "2019-03-20T15:48:26.022612: Epoch   0 Batch 1700/3125   train_loss = 1.063\n",
      "2019-03-20T15:48:26.629219: Epoch   0 Batch 1720/3125   train_loss = 1.129\n",
      "2019-03-20T15:48:27.247233: Epoch   0 Batch 1740/3125   train_loss = 1.190\n",
      "2019-03-20T15:48:27.834992: Epoch   0 Batch 1760/3125   train_loss = 1.331\n",
      "2019-03-20T15:48:28.448047: Epoch   0 Batch 1780/3125   train_loss = 1.094\n",
      "2019-03-20T15:48:29.108222: Epoch   0 Batch 1800/3125   train_loss = 1.171\n",
      "2019-03-20T15:48:29.754012: Epoch   0 Batch 1820/3125   train_loss = 1.193\n",
      "2019-03-20T15:48:30.419146: Epoch   0 Batch 1840/3125   train_loss = 1.270\n",
      "2019-03-20T15:48:31.027241: Epoch   0 Batch 1860/3125   train_loss = 1.294\n",
      "2019-03-20T15:48:31.629383: Epoch   0 Batch 1880/3125   train_loss = 1.217\n",
      "2019-03-20T15:48:32.226566: Epoch   0 Batch 1900/3125   train_loss = 1.058\n",
      "2019-03-20T15:48:32.878308: Epoch   0 Batch 1920/3125   train_loss = 1.126\n",
      "2019-03-20T15:48:33.522611: Epoch   0 Batch 1940/3125   train_loss = 1.186\n",
      "2019-03-20T15:48:34.165426: Epoch   0 Batch 1960/3125   train_loss = 1.183\n",
      "2019-03-20T15:48:34.746736: Epoch   0 Batch 1980/3125   train_loss = 1.124\n",
      "2019-03-20T15:48:35.368223: Epoch   0 Batch 2000/3125   train_loss = 1.382\n",
      "2019-03-20T15:48:35.958958: Epoch   0 Batch 2020/3125   train_loss = 1.281\n",
      "2019-03-20T15:48:36.620619: Epoch   0 Batch 2040/3125   train_loss = 1.141\n",
      "2019-03-20T15:48:37.400826: Epoch   0 Batch 2060/3125   train_loss = 1.060\n",
      "2019-03-20T15:48:38.103161: Epoch   0 Batch 2080/3125   train_loss = 1.284\n",
      "2019-03-20T15:48:38.835751: Epoch   0 Batch 2100/3125   train_loss = 1.117\n",
      "2019-03-20T15:48:39.544039: Epoch   0 Batch 2120/3125   train_loss = 1.105\n",
      "2019-03-20T15:48:40.277123: Epoch   0 Batch 2140/3125   train_loss = 1.163\n",
      "2019-03-20T15:48:40.922914: Epoch   0 Batch 2160/3125   train_loss = 1.116\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:48:41.543905: Epoch   0 Batch 2180/3125   train_loss = 1.134\n",
      "2019-03-20T15:48:42.158944: Epoch   0 Batch 2200/3125   train_loss = 1.136\n",
      "2019-03-20T15:48:42.777454: Epoch   0 Batch 2220/3125   train_loss = 1.104\n",
      "2019-03-20T15:48:43.378108: Epoch   0 Batch 2240/3125   train_loss = 1.061\n",
      "2019-03-20T15:48:43.959915: Epoch   0 Batch 2260/3125   train_loss = 1.186\n",
      "2019-03-20T15:48:44.549162: Epoch   0 Batch 2280/3125   train_loss = 1.192\n",
      "2019-03-20T15:48:45.198424: Epoch   0 Batch 2300/3125   train_loss = 1.165\n",
      "2019-03-20T15:48:45.858103: Epoch   0 Batch 2320/3125   train_loss = 1.354\n",
      "2019-03-20T15:48:46.500920: Epoch   0 Batch 2340/3125   train_loss = 1.175\n",
      "2019-03-20T15:48:47.184404: Epoch   0 Batch 2360/3125   train_loss = 1.175\n",
      "2019-03-20T15:48:47.811842: Epoch   0 Batch 2380/3125   train_loss = 1.069\n",
      "2019-03-20T15:48:48.487393: Epoch   0 Batch 2400/3125   train_loss = 1.270\n",
      "2019-03-20T15:48:49.095487: Epoch   0 Batch 2420/3125   train_loss = 1.067\n",
      "2019-03-20T15:48:49.675805: Epoch   0 Batch 2440/3125   train_loss = 1.151\n",
      "2019-03-20T15:48:50.368716: Epoch   0 Batch 2460/3125   train_loss = 1.085\n",
      "2019-03-20T15:48:51.004090: Epoch   0 Batch 2480/3125   train_loss = 1.204\n",
      "2019-03-20T15:48:51.625577: Epoch   0 Batch 2500/3125   train_loss = 1.154\n",
      "2019-03-20T15:48:52.279306: Epoch   0 Batch 2520/3125   train_loss = 1.053\n",
      "2019-03-20T15:48:53.016358: Epoch   0 Batch 2540/3125   train_loss = 1.002\n",
      "2019-03-20T15:48:53.621476: Epoch   0 Batch 2560/3125   train_loss = 0.906\n",
      "2019-03-20T15:48:54.229572: Epoch   0 Batch 2580/3125   train_loss = 1.094\n",
      "2019-03-20T15:48:54.834194: Epoch   0 Batch 2600/3125   train_loss = 1.194\n",
      "2019-03-20T15:48:55.529088: Epoch   0 Batch 2620/3125   train_loss = 1.080\n",
      "2019-03-20T15:48:56.209599: Epoch   0 Batch 2640/3125   train_loss = 1.121\n",
      "2019-03-20T15:48:56.857869: Epoch   0 Batch 2660/3125   train_loss = 1.207\n",
      "2019-03-20T15:48:57.518043: Epoch   0 Batch 2680/3125   train_loss = 1.050\n",
      "2019-03-20T15:48:58.182683: Epoch   0 Batch 2700/3125   train_loss = 1.198\n",
      "2019-03-20T15:48:58.790281: Epoch   0 Batch 2720/3125   train_loss = 1.149\n",
      "2019-03-20T15:48:59.426651: Epoch   0 Batch 2740/3125   train_loss = 1.182\n",
      "2019-03-20T15:49:00.068966: Epoch   0 Batch 2760/3125   train_loss = 1.203\n",
      "2019-03-20T15:49:00.691941: Epoch   0 Batch 2780/3125   train_loss = 1.007\n",
      "2019-03-20T15:49:01.371955: Epoch   0 Batch 2800/3125   train_loss = 1.423\n",
      "2019-03-20T15:49:02.113969: Epoch   0 Batch 2820/3125   train_loss = 1.441\n",
      "2019-03-20T15:49:02.801919: Epoch   0 Batch 2840/3125   train_loss = 1.230\n",
      "2019-03-20T15:49:03.463086: Epoch   0 Batch 2860/3125   train_loss = 1.147\n",
      "2019-03-20T15:49:04.082092: Epoch   0 Batch 2880/3125   train_loss = 1.184\n",
      "2019-03-20T15:49:04.696139: Epoch   0 Batch 2900/3125   train_loss = 1.137\n",
      "2019-03-20T15:49:05.327050: Epoch   0 Batch 2920/3125   train_loss = 1.255\n",
      "2019-03-20T15:49:05.954490: Epoch   0 Batch 2940/3125   train_loss = 1.125\n",
      "2019-03-20T15:49:06.574486: Epoch   0 Batch 2960/3125   train_loss = 1.222\n",
      "2019-03-20T15:49:07.197461: Epoch   0 Batch 2980/3125   train_loss = 1.163\n",
      "2019-03-20T15:49:07.838292: Epoch   0 Batch 3000/3125   train_loss = 1.203\n",
      "2019-03-20T15:49:08.438451: Epoch   0 Batch 3020/3125   train_loss = 1.222\n",
      "2019-03-20T15:49:09.106561: Epoch   0 Batch 3040/3125   train_loss = 1.091\n",
      "2019-03-20T15:49:09.805423: Epoch   0 Batch 3060/3125   train_loss = 1.206\n",
      "2019-03-20T15:49:10.463118: Epoch   0 Batch 3080/3125   train_loss = 1.303\n",
      "2019-03-20T15:49:11.116844: Epoch   0 Batch 3100/3125   train_loss = 1.161\n",
      "2019-03-20T15:49:11.793388: Epoch   0 Batch 3120/3125   train_loss = 1.063\n",
      "2019-03-20T15:49:12.051482: Epoch   0 Batch    0/781   test_loss = 0.978\n",
      "2019-03-20T15:49:12.249386: Epoch   0 Batch   20/781   test_loss = 1.119\n",
      "2019-03-20T15:49:12.431913: Epoch   0 Batch   40/781   test_loss = 1.037\n",
      "2019-03-20T15:49:12.622379: Epoch   0 Batch   60/781   test_loss = 1.365\n",
      "2019-03-20T15:49:12.810858: Epoch   0 Batch   80/781   test_loss = 1.320\n",
      "2019-03-20T15:49:12.993880: Epoch   0 Batch  100/781   test_loss = 1.314\n",
      "2019-03-20T15:49:13.167480: Epoch   0 Batch  120/781   test_loss = 1.236\n",
      "2019-03-20T15:49:13.337112: Epoch   0 Batch  140/781   test_loss = 1.174\n",
      "2019-03-20T15:49:13.506248: Epoch   0 Batch  160/781   test_loss = 1.337\n",
      "2019-03-20T15:49:13.672407: Epoch   0 Batch  180/781   test_loss = 1.210\n",
      "2019-03-20T15:49:13.845511: Epoch   0 Batch  200/781   test_loss = 1.162\n",
      "2019-03-20T15:49:14.021095: Epoch   0 Batch  220/781   test_loss = 0.956\n",
      "2019-03-20T15:49:14.200150: Epoch   0 Batch  240/781   test_loss = 1.164\n",
      "2019-03-20T15:49:14.381190: Epoch   0 Batch  260/781   test_loss = 1.133\n",
      "2019-03-20T15:49:14.559253: Epoch   0 Batch  280/781   test_loss = 1.434\n",
      "2019-03-20T15:49:14.732356: Epoch   0 Batch  300/781   test_loss = 1.139\n",
      "2019-03-20T15:49:14.901988: Epoch   0 Batch  320/781   test_loss = 1.296\n",
      "2019-03-20T15:49:15.083104: Epoch   0 Batch  340/781   test_loss = 0.850\n",
      "2019-03-20T15:49:15.250252: Epoch   0 Batch  360/781   test_loss = 1.357\n",
      "2019-03-20T15:49:15.419391: Epoch   0 Batch  380/781   test_loss = 1.141\n",
      "2019-03-20T15:49:15.590011: Epoch   0 Batch  400/781   test_loss = 1.080\n",
      "2019-03-20T15:49:15.777003: Epoch   0 Batch  420/781   test_loss = 1.062\n",
      "2019-03-20T15:49:15.949611: Epoch   0 Batch  440/781   test_loss = 1.195\n",
      "2019-03-20T15:49:16.122715: Epoch   0 Batch  460/781   test_loss = 1.100\n",
      "2019-03-20T15:49:16.295817: Epoch   0 Batch  480/781   test_loss = 1.088\n",
      "2019-03-20T15:49:16.466440: Epoch   0 Batch  500/781   test_loss = 0.904\n",
      "2019-03-20T15:49:16.636570: Epoch   0 Batch  520/781   test_loss = 1.174\n",
      "2019-03-20T15:49:16.822568: Epoch   0 Batch  540/781   test_loss = 0.972\n",
      "2019-03-20T15:49:17.016503: Epoch   0 Batch  560/781   test_loss = 1.277\n",
      "2019-03-20T15:49:17.217383: Epoch   0 Batch  580/781   test_loss = 1.154\n",
      "2019-03-20T15:49:17.397431: Epoch   0 Batch  600/781   test_loss = 1.182\n",
      "2019-03-20T15:49:17.574006: Epoch   0 Batch  620/781   test_loss = 1.185\n",
      "2019-03-20T15:49:17.746615: Epoch   0 Batch  640/781   test_loss = 1.208\n",
      "2019-03-20T15:49:17.919223: Epoch   0 Batch  660/781   test_loss = 1.118\n",
      "2019-03-20T15:49:18.093317: Epoch   0 Batch  680/781   test_loss = 1.350\n",
      "2019-03-20T15:49:18.266421: Epoch   0 Batch  700/781   test_loss = 1.124\n",
      "2019-03-20T15:49:18.440516: Epoch   0 Batch  720/781   test_loss = 1.309\n",
      "2019-03-20T15:49:18.618580: Epoch   0 Batch  740/781   test_loss = 1.165\n",
      "2019-03-20T15:49:18.788707: Epoch   0 Batch  760/781   test_loss = 1.102\n",
      "2019-03-20T15:49:18.961316: Epoch   0 Batch  780/781   test_loss = 1.135\n",
      "2019-03-20T15:49:19.808982: Epoch   1 Batch   15/3125   train_loss = 1.175\n",
      "2019-03-20T15:49:20.392768: Epoch   1 Batch   35/3125   train_loss = 1.100\n",
      "2019-03-20T15:49:20.978048: Epoch   1 Batch   55/3125   train_loss = 1.223\n",
      "2019-03-20T15:49:21.567293: Epoch   1 Batch   75/3125   train_loss = 1.112\n",
      "2019-03-20T15:49:22.200684: Epoch   1 Batch   95/3125   train_loss = 0.990\n",
      "2019-03-20T15:49:22.785466: Epoch   1 Batch  115/3125   train_loss = 1.220\n",
      "2019-03-20T15:49:23.398024: Epoch   1 Batch  135/3125   train_loss = 0.978\n",
      "2019-03-20T15:49:23.972888: Epoch   1 Batch  155/3125   train_loss = 1.106\n",
      "2019-03-20T15:49:24.549239: Epoch   1 Batch  175/3125   train_loss = 1.076\n",
      "2019-03-20T15:49:25.121622: Epoch   1 Batch  195/3125   train_loss = 1.219\n",
      "2019-03-20T15:49:25.696980: Epoch   1 Batch  215/3125   train_loss = 1.147\n",
      "2019-03-20T15:49:26.307554: Epoch   1 Batch  235/3125   train_loss = 1.042\n",
      "2019-03-20T15:49:26.926066: Epoch   1 Batch  255/3125   train_loss = 1.199\n",
      "2019-03-20T15:49:27.587728: Epoch   1 Batch  275/3125   train_loss = 1.021\n",
      "2019-03-20T15:49:28.226574: Epoch   1 Batch  295/3125   train_loss = 0.984\n",
      "2019-03-20T15:49:28.876337: Epoch   1 Batch  315/3125   train_loss = 1.064\n",
      "2019-03-20T15:49:29.487899: Epoch   1 Batch  335/3125   train_loss = 0.978\n",
      "2019-03-20T15:49:30.055322: Epoch   1 Batch  355/3125   train_loss = 1.072\n",
      "2019-03-20T15:49:30.658415: Epoch   1 Batch  375/3125   train_loss = 1.230\n",
      "2019-03-20T15:49:31.239230: Epoch   1 Batch  395/3125   train_loss = 1.072\n",
      "2019-03-20T15:49:31.856750: Epoch   1 Batch  415/3125   train_loss = 1.216\n",
      "2019-03-20T15:49:32.476251: Epoch   1 Batch  435/3125   train_loss = 1.162\n",
      "2019-03-20T15:49:33.107162: Epoch   1 Batch  455/3125   train_loss = 1.092\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:49:33.683016: Epoch   1 Batch  475/3125   train_loss = 1.181\n",
      "2019-03-20T15:49:34.251926: Epoch   1 Batch  495/3125   train_loss = 1.023\n",
      "2019-03-20T15:49:34.843654: Epoch   1 Batch  515/3125   train_loss = 1.201\n",
      "2019-03-20T15:49:35.506804: Epoch   1 Batch  535/3125   train_loss = 1.148\n",
      "2019-03-20T15:49:36.088114: Epoch   1 Batch  555/3125   train_loss = 1.200\n",
      "2019-03-20T15:49:36.655041: Epoch   1 Batch  575/3125   train_loss = 1.091\n",
      "2019-03-20T15:49:37.233872: Epoch   1 Batch  595/3125   train_loss = 1.388\n",
      "2019-03-20T15:49:37.815679: Epoch   1 Batch  615/3125   train_loss = 1.106\n",
      "2019-03-20T15:49:38.431709: Epoch   1 Batch  635/3125   train_loss = 1.233\n",
      "2019-03-20T15:49:39.008059: Epoch   1 Batch  655/3125   train_loss = 0.986\n",
      "2019-03-20T15:49:39.579450: Epoch   1 Batch  675/3125   train_loss = 0.943\n",
      "2019-03-20T15:49:40.152825: Epoch   1 Batch  695/3125   train_loss = 1.044\n",
      "2019-03-20T15:49:40.719752: Epoch   1 Batch  715/3125   train_loss = 1.133\n",
      "2019-03-20T15:49:41.331815: Epoch   1 Batch  735/3125   train_loss = 0.939\n",
      "2019-03-20T15:49:41.904693: Epoch   1 Batch  755/3125   train_loss = 1.112\n",
      "2019-03-20T15:49:42.483028: Epoch   1 Batch  775/3125   train_loss = 1.027\n",
      "2019-03-20T15:49:43.066322: Epoch   1 Batch  795/3125   train_loss = 1.246\n",
      "2019-03-20T15:49:43.634737: Epoch   1 Batch  815/3125   train_loss = 1.157\n",
      "2019-03-20T15:49:44.240351: Epoch   1 Batch  835/3125   train_loss = 1.106\n",
      "2019-03-20T15:49:44.828110: Epoch   1 Batch  855/3125   train_loss = 1.291\n",
      "2019-03-20T15:49:45.409420: Epoch   1 Batch  875/3125   train_loss = 1.212\n",
      "2019-03-20T15:49:45.987260: Epoch   1 Batch  895/3125   train_loss = 1.066\n",
      "2019-03-20T15:49:46.566091: Epoch   1 Batch  915/3125   train_loss = 1.069\n",
      "2019-03-20T15:49:47.182617: Epoch   1 Batch  935/3125   train_loss = 1.233\n",
      "2019-03-20T15:49:47.774344: Epoch   1 Batch  955/3125   train_loss = 1.131\n",
      "2019-03-20T15:49:48.354662: Epoch   1 Batch  975/3125   train_loss = 1.145\n",
      "2019-03-20T15:49:48.938949: Epoch   1 Batch  995/3125   train_loss = 0.926\n",
      "2019-03-20T15:49:49.520259: Epoch   1 Batch 1015/3125   train_loss = 1.101\n",
      "2019-03-20T15:49:50.151666: Epoch   1 Batch 1035/3125   train_loss = 1.133\n",
      "2019-03-20T15:49:50.749345: Epoch   1 Batch 1055/3125   train_loss = 1.080\n",
      "2019-03-20T15:49:51.363886: Epoch   1 Batch 1075/3125   train_loss = 1.040\n",
      "2019-03-20T15:49:51.985374: Epoch   1 Batch 1095/3125   train_loss = 1.020\n",
      "2019-03-20T15:49:52.662909: Epoch   1 Batch 1115/3125   train_loss = 1.077\n",
      "2019-03-20T15:49:53.328042: Epoch   1 Batch 1135/3125   train_loss = 1.033\n",
      "2019-03-20T15:49:53.954489: Epoch   1 Batch 1155/3125   train_loss = 1.166\n",
      "2019-03-20T15:49:54.624584: Epoch   1 Batch 1175/3125   train_loss = 1.100\n",
      "2019-03-20T15:49:55.292198: Epoch   1 Batch 1195/3125   train_loss = 1.175\n",
      "2019-03-20T15:49:55.895828: Epoch   1 Batch 1215/3125   train_loss = 0.968\n",
      "2019-03-20T15:49:56.483092: Epoch   1 Batch 1235/3125   train_loss = 1.096\n",
      "2019-03-20T15:49:57.089698: Epoch   1 Batch 1255/3125   train_loss = 0.995\n",
      "2019-03-20T15:49:57.687873: Epoch   1 Batch 1275/3125   train_loss = 1.023\n",
      "2019-03-20T15:49:58.307871: Epoch   1 Batch 1295/3125   train_loss = 1.122\n",
      "2019-03-20T15:49:59.014669: Epoch   1 Batch 1315/3125   train_loss = 1.194\n",
      "2019-03-20T15:49:59.646077: Epoch   1 Batch 1335/3125   train_loss = 1.070\n",
      "2019-03-20T15:50:00.260619: Epoch   1 Batch 1355/3125   train_loss = 1.087\n",
      "2019-03-20T15:50:00.899962: Epoch   1 Batch 1375/3125   train_loss = 1.156\n",
      "2019-03-20T15:50:01.598825: Epoch   1 Batch 1395/3125   train_loss = 1.027\n",
      "2019-03-20T15:50:02.301158: Epoch   1 Batch 1415/3125   train_loss = 1.112\n",
      "2019-03-20T15:50:02.945461: Epoch   1 Batch 1435/3125   train_loss = 1.262\n",
      "2019-03-20T15:50:03.585298: Epoch   1 Batch 1455/3125   train_loss = 1.222\n",
      "2019-03-20T15:50:04.252914: Epoch   1 Batch 1475/3125   train_loss = 1.116\n",
      "2019-03-20T15:50:04.923505: Epoch   1 Batch 1495/3125   train_loss = 1.020\n",
      "2019-03-20T15:50:05.619885: Epoch   1 Batch 1515/3125   train_loss = 0.970\n",
      "2019-03-20T15:50:06.283533: Epoch   1 Batch 1535/3125   train_loss = 0.936\n",
      "2019-03-20T15:50:06.965531: Epoch   1 Batch 1555/3125   train_loss = 1.085\n",
      "2019-03-20T15:50:07.738794: Epoch   1 Batch 1575/3125   train_loss = 1.030\n",
      "2019-03-20T15:50:08.431208: Epoch   1 Batch 1595/3125   train_loss = 1.154\n",
      "2019-03-20T15:50:09.088409: Epoch   1 Batch 1615/3125   train_loss = 1.002\n",
      "2019-03-20T15:50:09.737174: Epoch   1 Batch 1635/3125   train_loss = 1.051\n",
      "2019-03-20T15:50:10.401811: Epoch   1 Batch 1655/3125   train_loss = 1.144\n",
      "2019-03-20T15:50:11.056034: Epoch   1 Batch 1675/3125   train_loss = 0.984\n",
      "2019-03-20T15:50:11.711744: Epoch   1 Batch 1695/3125   train_loss = 1.035\n",
      "2019-03-20T15:50:12.350591: Epoch   1 Batch 1715/3125   train_loss = 1.020\n",
      "2019-03-20T15:50:13.021676: Epoch   1 Batch 1735/3125   train_loss = 1.209\n",
      "2019-03-20T15:50:13.690283: Epoch   1 Batch 1755/3125   train_loss = 1.030\n",
      "2019-03-20T15:50:14.334089: Epoch   1 Batch 1775/3125   train_loss = 1.047\n",
      "2019-03-20T15:50:14.973928: Epoch   1 Batch 1795/3125   train_loss = 1.047\n",
      "2019-03-20T15:50:15.630631: Epoch   1 Batch 1815/3125   train_loss = 1.051\n",
      "2019-03-20T15:50:16.340406: Epoch   1 Batch 1835/3125   train_loss = 1.117\n",
      "2019-03-20T15:50:17.016948: Epoch   1 Batch 1855/3125   train_loss = 0.954\n",
      "2019-03-20T15:50:17.644386: Epoch   1 Batch 1875/3125   train_loss = 1.137\n",
      "2019-03-20T15:50:18.286210: Epoch   1 Batch 1895/3125   train_loss = 1.061\n",
      "2019-03-20T15:50:18.936464: Epoch   1 Batch 1915/3125   train_loss = 0.920\n",
      "2019-03-20T15:50:19.596639: Epoch   1 Batch 1935/3125   train_loss = 1.052\n",
      "2019-03-20T15:50:20.379325: Epoch   1 Batch 1955/3125   train_loss = 0.951\n",
      "2019-03-20T15:50:21.060330: Epoch   1 Batch 1975/3125   train_loss = 1.044\n",
      "2019-03-20T15:50:21.720008: Epoch   1 Batch 1995/3125   train_loss = 1.221\n",
      "2019-03-20T15:50:22.344967: Epoch   1 Batch 2015/3125   train_loss = 1.073\n",
      "2019-03-20T15:50:22.986791: Epoch   1 Batch 2035/3125   train_loss = 1.147\n",
      "2019-03-20T15:50:23.620181: Epoch   1 Batch 2055/3125   train_loss = 0.941\n",
      "2019-03-20T15:50:24.266466: Epoch   1 Batch 2075/3125   train_loss = 1.165\n",
      "2019-03-20T15:50:24.898371: Epoch   1 Batch 2095/3125   train_loss = 1.020\n",
      "2019-03-20T15:50:25.536224: Epoch   1 Batch 2115/3125   train_loss = 1.150\n",
      "2019-03-20T15:50:26.194912: Epoch   1 Batch 2135/3125   train_loss = 0.994\n",
      "2019-03-20T15:50:26.841197: Epoch   1 Batch 2155/3125   train_loss = 1.043\n",
      "2019-03-20T15:50:27.498892: Epoch   1 Batch 2175/3125   train_loss = 1.048\n",
      "2019-03-20T15:50:28.156587: Epoch   1 Batch 2195/3125   train_loss = 1.085\n",
      "2019-03-20T15:50:28.799898: Epoch   1 Batch 2215/3125   train_loss = 1.094\n",
      "2019-03-20T15:50:29.464535: Epoch   1 Batch 2235/3125   train_loss = 1.091\n",
      "2019-03-20T15:50:30.161413: Epoch   1 Batch 2255/3125   train_loss = 1.156\n",
      "2019-03-20T15:50:30.843411: Epoch   1 Batch 2275/3125   train_loss = 0.875\n",
      "2019-03-20T15:50:31.644450: Epoch   1 Batch 2295/3125   train_loss = 1.226\n",
      "2019-03-20T15:50:32.433088: Epoch   1 Batch 2315/3125   train_loss = 1.195\n",
      "2019-03-20T15:50:33.157247: Epoch   1 Batch 2335/3125   train_loss = 1.102\n",
      "2019-03-20T15:50:33.829325: Epoch   1 Batch 2355/3125   train_loss = 1.150\n",
      "2019-03-20T15:50:34.484048: Epoch   1 Batch 2375/3125   train_loss = 1.210\n",
      "2019-03-20T15:50:35.155132: Epoch   1 Batch 2395/3125   train_loss = 1.022\n",
      "2019-03-20T15:50:35.807368: Epoch   1 Batch 2415/3125   train_loss = 1.005\n",
      "2019-03-20T15:50:36.466551: Epoch   1 Batch 2435/3125   train_loss = 1.048\n",
      "2019-03-20T15:50:37.136150: Epoch   1 Batch 2455/3125   train_loss = 1.107\n",
      "2019-03-20T15:50:37.814676: Epoch   1 Batch 2475/3125   train_loss = 1.077\n",
      "2019-03-20T15:50:38.471876: Epoch   1 Batch 2495/3125   train_loss = 1.010\n",
      "2019-03-20T15:50:39.143456: Epoch   1 Batch 2515/3125   train_loss = 1.073\n",
      "2019-03-20T15:50:39.824960: Epoch   1 Batch 2535/3125   train_loss = 1.101\n",
      "2019-03-20T15:50:40.506461: Epoch   1 Batch 2555/3125   train_loss = 0.928\n",
      "2019-03-20T15:50:41.191933: Epoch   1 Batch 2575/3125   train_loss = 0.953\n",
      "2019-03-20T15:50:41.867979: Epoch   1 Batch 2595/3125   train_loss = 1.011\n",
      "2019-03-20T15:50:42.536585: Epoch   1 Batch 2615/3125   train_loss = 1.091\n",
      "2019-03-20T15:50:43.215111: Epoch   1 Batch 2635/3125   train_loss = 0.956\n",
      "2019-03-20T15:50:43.895621: Epoch   1 Batch 2655/3125   train_loss = 1.012\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:50:44.560757: Epoch   1 Batch 2675/3125   train_loss = 0.984\n",
      "2019-03-20T15:50:45.229363: Epoch   1 Batch 2695/3125   train_loss = 1.026\n",
      "2019-03-20T15:50:45.904417: Epoch   1 Batch 2715/3125   train_loss = 1.037\n",
      "2019-03-20T15:50:46.593360: Epoch   1 Batch 2735/3125   train_loss = 0.830\n",
      "2019-03-20T15:50:47.268910: Epoch   1 Batch 2755/3125   train_loss = 1.073\n",
      "2019-03-20T15:50:47.934044: Epoch   1 Batch 2775/3125   train_loss = 1.118\n",
      "2019-03-20T15:50:48.615051: Epoch   1 Batch 2795/3125   train_loss = 1.035\n",
      "2019-03-20T15:50:49.260842: Epoch   1 Batch 2815/3125   train_loss = 1.010\n",
      "2019-03-20T15:50:49.940856: Epoch   1 Batch 2835/3125   train_loss = 1.092\n",
      "2019-03-20T15:50:50.604007: Epoch   1 Batch 2855/3125   train_loss = 1.095\n",
      "2019-03-20T15:50:51.281541: Epoch   1 Batch 2875/3125   train_loss = 1.015\n",
      "2019-03-20T15:50:51.979908: Epoch   1 Batch 2895/3125   train_loss = 1.033\n",
      "2019-03-20T15:50:52.651984: Epoch   1 Batch 2915/3125   train_loss = 1.062\n",
      "2019-03-20T15:50:53.326544: Epoch   1 Batch 2935/3125   train_loss = 1.109\n",
      "2019-03-20T15:50:54.100302: Epoch   1 Batch 2955/3125   train_loss = 1.178\n",
      "2019-03-20T15:50:54.795196: Epoch   1 Batch 2975/3125   train_loss = 1.014\n",
      "2019-03-20T15:50:55.486618: Epoch   1 Batch 2995/3125   train_loss = 0.992\n",
      "2019-03-20T15:50:56.175066: Epoch   1 Batch 3015/3125   train_loss = 0.968\n",
      "2019-03-20T15:50:56.845163: Epoch   1 Batch 3035/3125   train_loss = 1.052\n",
      "2019-03-20T15:50:57.540551: Epoch   1 Batch 3055/3125   train_loss = 1.042\n",
      "2019-03-20T15:50:58.259252: Epoch   1 Batch 3075/3125   train_loss = 1.026\n",
      "2019-03-20T15:50:58.877763: Epoch   1 Batch 3095/3125   train_loss = 1.000\n",
      "2019-03-20T15:50:59.506689: Epoch   1 Batch 3115/3125   train_loss = 0.830\n",
      "2019-03-20T15:51:00.018066: Epoch   1 Batch   19/781   test_loss = 1.012\n",
      "2019-03-20T15:51:00.207536: Epoch   1 Batch   39/781   test_loss = 0.872\n",
      "2019-03-20T15:51:00.434208: Epoch   1 Batch   59/781   test_loss = 0.912\n",
      "2019-03-20T15:51:00.629135: Epoch   1 Batch   79/781   test_loss = 0.997\n",
      "2019-03-20T15:51:00.825582: Epoch   1 Batch   99/781   test_loss = 1.013\n",
      "2019-03-20T15:51:01.023456: Epoch   1 Batch  119/781   test_loss = 0.907\n",
      "2019-03-20T15:51:01.214911: Epoch   1 Batch  139/781   test_loss = 1.039\n",
      "2019-03-20T15:51:01.417278: Epoch   1 Batch  159/781   test_loss = 1.018\n",
      "2019-03-20T15:51:01.603774: Epoch   1 Batch  179/781   test_loss = 0.889\n",
      "2019-03-20T15:51:01.803663: Epoch   1 Batch  199/781   test_loss = 0.899\n",
      "2019-03-20T15:51:01.998589: Epoch   1 Batch  219/781   test_loss = 1.057\n",
      "2019-03-20T15:51:02.201453: Epoch   1 Batch  239/781   test_loss = 1.179\n",
      "2019-03-20T15:51:02.408780: Epoch   1 Batch  259/781   test_loss = 1.017\n",
      "2019-03-20T15:51:02.604700: Epoch   1 Batch  279/781   test_loss = 1.122\n",
      "2019-03-20T15:51:02.785738: Epoch   1 Batch  299/781   test_loss = 1.132\n",
      "2019-03-20T15:51:02.962812: Epoch   1 Batch  319/781   test_loss = 0.976\n",
      "2019-03-20T15:51:03.174109: Epoch   1 Batch  339/781   test_loss = 0.961\n",
      "2019-03-20T15:51:03.364072: Epoch   1 Batch  359/781   test_loss = 0.980\n",
      "2019-03-20T15:51:03.560985: Epoch   1 Batch  379/781   test_loss = 1.022\n",
      "2019-03-20T15:51:03.759386: Epoch   1 Batch  399/781   test_loss = 0.856\n",
      "2019-03-20T15:51:03.953319: Epoch   1 Batch  419/781   test_loss = 0.975\n",
      "2019-03-20T15:51:04.144775: Epoch   1 Batch  439/781   test_loss = 1.018\n",
      "2019-03-20T15:51:04.327798: Epoch   1 Batch  459/781   test_loss = 1.043\n",
      "2019-03-20T15:51:04.508342: Epoch   1 Batch  479/781   test_loss = 1.056\n",
      "2019-03-20T15:51:04.690374: Epoch   1 Batch  499/781   test_loss = 0.943\n",
      "2019-03-20T15:51:04.884314: Epoch   1 Batch  519/781   test_loss = 1.071\n",
      "2019-03-20T15:51:05.086181: Epoch   1 Batch  539/781   test_loss = 0.858\n",
      "2019-03-20T15:51:05.290537: Epoch   1 Batch  559/781   test_loss = 1.151\n",
      "2019-03-20T15:51:05.480501: Epoch   1 Batch  579/781   test_loss = 1.077\n",
      "2019-03-20T15:51:05.664019: Epoch   1 Batch  599/781   test_loss = 0.973\n",
      "2019-03-20T15:51:05.849026: Epoch   1 Batch  619/781   test_loss = 1.118\n",
      "2019-03-20T15:51:06.051893: Epoch   1 Batch  639/781   test_loss = 0.882\n",
      "2019-03-20T15:51:06.254261: Epoch   1 Batch  659/781   test_loss = 1.083\n",
      "2019-03-20T15:51:06.479441: Epoch   1 Batch  679/781   test_loss = 1.203\n",
      "2019-03-20T15:51:06.690739: Epoch   1 Batch  699/781   test_loss = 0.870\n",
      "2019-03-20T15:51:06.890130: Epoch   1 Batch  719/781   test_loss = 1.005\n",
      "2019-03-20T15:51:07.103408: Epoch   1 Batch  739/781   test_loss = 0.984\n",
      "2019-03-20T15:51:07.306273: Epoch   1 Batch  759/781   test_loss = 0.912\n",
      "2019-03-20T15:51:07.502687: Epoch   1 Batch  779/781   test_loss = 0.793\n",
      "2019-03-20T15:51:08.422272: Epoch   2 Batch   10/3125   train_loss = 0.951\n",
      "2019-03-20T15:51:09.144444: Epoch   2 Batch   30/3125   train_loss = 1.006\n",
      "2019-03-20T15:51:09.813050: Epoch   2 Batch   50/3125   train_loss = 1.140\n",
      "2019-03-20T15:51:10.482153: Epoch   2 Batch   70/3125   train_loss = 1.059\n",
      "2019-03-20T15:51:11.145799: Epoch   2 Batch   90/3125   train_loss = 1.079\n",
      "2019-03-20T15:51:11.782661: Epoch   2 Batch  110/3125   train_loss = 0.954\n",
      "2019-03-20T15:51:12.393731: Epoch   2 Batch  130/3125   train_loss = 0.972\n",
      "2019-03-20T15:51:12.982979: Epoch   2 Batch  150/3125   train_loss = 1.165\n",
      "2019-03-20T15:51:13.563297: Epoch   2 Batch  170/3125   train_loss = 1.012\n",
      "2019-03-20T15:51:14.174863: Epoch   2 Batch  190/3125   train_loss = 1.084\n",
      "2019-03-20T15:51:14.776014: Epoch   2 Batch  210/3125   train_loss = 1.004\n",
      "2019-03-20T15:51:15.366252: Epoch   2 Batch  230/3125   train_loss = 1.056\n",
      "2019-03-20T15:51:15.977819: Epoch   2 Batch  250/3125   train_loss = 0.958\n",
      "2019-03-20T15:51:16.607242: Epoch   2 Batch  270/3125   train_loss = 0.815\n",
      "2019-03-20T15:51:17.243609: Epoch   2 Batch  290/3125   train_loss = 1.070\n",
      "2019-03-20T15:51:17.831368: Epoch   2 Batch  310/3125   train_loss = 1.034\n",
      "2019-03-20T15:51:18.454342: Epoch   2 Batch  330/3125   train_loss = 1.029\n",
      "2019-03-20T15:51:19.098148: Epoch   2 Batch  350/3125   train_loss = 0.933\n",
      "2019-03-20T15:51:19.755347: Epoch   2 Batch  370/3125   train_loss = 1.070\n",
      "2019-03-20T15:51:20.397172: Epoch   2 Batch  390/3125   train_loss = 1.195\n",
      "2019-03-20T15:51:21.046432: Epoch   2 Batch  410/3125   train_loss = 0.931\n",
      "2019-03-20T15:51:21.664942: Epoch   2 Batch  430/3125   train_loss = 1.085\n",
      "2019-03-20T15:51:22.284941: Epoch   2 Batch  450/3125   train_loss = 0.965\n",
      "2019-03-20T15:51:22.895515: Epoch   2 Batch  470/3125   train_loss = 0.974\n",
      "2019-03-20T15:51:23.506090: Epoch   2 Batch  490/3125   train_loss = 1.004\n",
      "2019-03-20T15:51:24.119144: Epoch   2 Batch  510/3125   train_loss = 1.057\n",
      "2019-03-20T15:51:24.725751: Epoch   2 Batch  530/3125   train_loss = 0.991\n",
      "2019-03-20T15:51:25.350214: Epoch   2 Batch  550/3125   train_loss = 0.974\n",
      "2019-03-20T15:51:26.006916: Epoch   2 Batch  570/3125   train_loss = 1.100\n",
      "2019-03-20T15:51:26.668083: Epoch   2 Batch  590/3125   train_loss = 1.011\n",
      "2019-03-20T15:51:27.318337: Epoch   2 Batch  610/3125   train_loss = 0.971\n",
      "2019-03-20T15:51:27.979999: Epoch   2 Batch  630/3125   train_loss = 1.070\n",
      "2019-03-20T15:51:28.618846: Epoch   2 Batch  650/3125   train_loss = 1.055\n",
      "2019-03-20T15:51:29.254718: Epoch   2 Batch  670/3125   train_loss = 0.947\n",
      "2019-03-20T15:51:29.905468: Epoch   2 Batch  690/3125   train_loss = 0.910\n",
      "2019-03-20T15:51:30.606234: Epoch   2 Batch  710/3125   train_loss = 0.941\n",
      "2019-03-20T15:51:31.308073: Epoch   2 Batch  730/3125   train_loss = 0.879\n",
      "2019-03-20T15:51:31.998998: Epoch   2 Batch  750/3125   train_loss = 0.987\n",
      "2019-03-20T15:51:32.655206: Epoch   2 Batch  770/3125   train_loss = 0.919\n",
      "2019-03-20T15:51:33.334722: Epoch   2 Batch  790/3125   train_loss = 0.932\n",
      "2019-03-20T15:51:33.973076: Epoch   2 Batch  810/3125   train_loss = 0.839\n",
      "2019-03-20T15:51:34.597041: Epoch   2 Batch  830/3125   train_loss = 0.839\n",
      "2019-03-20T15:51:35.250270: Epoch   2 Batch  850/3125   train_loss = 1.028\n",
      "2019-03-20T15:51:35.912428: Epoch   2 Batch  870/3125   train_loss = 0.933\n",
      "2019-03-20T15:51:36.584012: Epoch   2 Batch  890/3125   train_loss = 0.946\n",
      "2019-03-20T15:51:37.230794: Epoch   2 Batch  910/3125   train_loss = 1.024\n",
      "2019-03-20T15:51:37.898412: Epoch   2 Batch  930/3125   train_loss = 1.053\n",
      "2019-03-20T15:51:38.555607: Epoch   2 Batch  950/3125   train_loss = 1.008\n",
      "2019-03-20T15:51:39.229669: Epoch   2 Batch  970/3125   train_loss = 1.058\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:51:39.930020: Epoch   2 Batch  990/3125   train_loss = 0.923\n",
      "2019-03-20T15:51:40.590195: Epoch   2 Batch 1010/3125   train_loss = 1.123\n",
      "2019-03-20T15:51:41.244912: Epoch   2 Batch 1030/3125   train_loss = 0.980\n",
      "2019-03-20T15:51:41.880784: Epoch   2 Batch 1050/3125   train_loss = 0.904\n",
      "2019-03-20T15:51:42.492350: Epoch   2 Batch 1070/3125   train_loss = 0.990\n",
      "2019-03-20T15:51:43.207085: Epoch   2 Batch 1090/3125   train_loss = 1.027\n",
      "2019-03-20T15:51:43.915371: Epoch   2 Batch 1110/3125   train_loss = 1.099\n",
      "2019-03-20T15:51:44.559176: Epoch   2 Batch 1130/3125   train_loss = 0.944\n",
      "2019-03-20T15:51:45.197031: Epoch   2 Batch 1150/3125   train_loss = 0.966\n",
      "2019-03-20T15:51:45.865639: Epoch   2 Batch 1170/3125   train_loss = 0.963\n",
      "2019-03-20T15:51:46.507956: Epoch   2 Batch 1190/3125   train_loss = 1.030\n",
      "2019-03-20T15:51:47.141843: Epoch   2 Batch 1210/3125   train_loss = 0.855\n",
      "2019-03-20T15:51:47.766306: Epoch   2 Batch 1230/3125   train_loss = 0.844\n",
      "2019-03-20T15:51:48.396720: Epoch   2 Batch 1250/3125   train_loss = 0.962\n",
      "2019-03-20T15:51:49.042511: Epoch   2 Batch 1270/3125   train_loss = 0.937\n",
      "2019-03-20T15:51:49.645149: Epoch   2 Batch 1290/3125   train_loss = 0.904\n",
      "2019-03-20T15:51:50.238860: Epoch   2 Batch 1310/3125   train_loss = 0.911\n",
      "2019-03-20T15:51:50.830587: Epoch   2 Batch 1330/3125   train_loss = 1.133\n",
      "2019-03-20T15:51:51.438184: Epoch   2 Batch 1350/3125   train_loss = 0.901\n",
      "2019-03-20T15:51:52.031400: Epoch   2 Batch 1370/3125   train_loss = 0.898\n",
      "2019-03-20T15:51:52.697030: Epoch   2 Batch 1390/3125   train_loss = 1.019\n",
      "2019-03-20T15:51:53.303141: Epoch   2 Batch 1410/3125   train_loss = 0.959\n",
      "2019-03-20T15:51:53.918179: Epoch   2 Batch 1430/3125   train_loss = 1.056\n",
      "2019-03-20T15:51:54.530738: Epoch   2 Batch 1450/3125   train_loss = 0.982\n",
      "2019-03-20T15:51:55.183968: Epoch   2 Batch 1470/3125   train_loss = 0.954\n",
      "2019-03-20T15:51:55.777184: Epoch   2 Batch 1490/3125   train_loss = 1.009\n",
      "2019-03-20T15:51:56.372382: Epoch   2 Batch 1510/3125   train_loss = 0.956\n",
      "2019-03-20T15:51:56.950221: Epoch   2 Batch 1530/3125   train_loss = 1.087\n",
      "2019-03-20T15:51:57.516651: Epoch   2 Batch 1550/3125   train_loss = 0.932\n",
      "2019-03-20T15:51:58.090026: Epoch   2 Batch 1570/3125   train_loss = 0.936\n",
      "2019-03-20T15:51:58.657944: Epoch   2 Batch 1590/3125   train_loss = 0.974\n",
      "2019-03-20T15:51:59.253143: Epoch   2 Batch 1610/3125   train_loss = 1.071\n",
      "2019-03-20T15:51:59.831478: Epoch   2 Batch 1630/3125   train_loss = 0.959\n",
      "2019-03-20T15:52:00.411301: Epoch   2 Batch 1650/3125   train_loss = 0.853\n",
      "2019-03-20T15:52:01.029810: Epoch   2 Batch 1670/3125   train_loss = 0.856\n",
      "2019-03-20T15:52:01.696929: Epoch   2 Batch 1690/3125   train_loss = 1.020\n",
      "2019-03-20T15:52:02.352144: Epoch   2 Batch 1710/3125   train_loss = 0.937\n",
      "2019-03-20T15:52:02.954782: Epoch   2 Batch 1730/3125   train_loss = 0.999\n",
      "2019-03-20T15:52:03.527661: Epoch   2 Batch 1750/3125   train_loss = 0.883\n",
      "2019-03-20T15:52:04.146668: Epoch   2 Batch 1770/3125   train_loss = 1.145\n",
      "2019-03-20T15:52:04.736410: Epoch   2 Batch 1790/3125   train_loss = 0.957\n",
      "2019-03-20T15:52:05.352937: Epoch   2 Batch 1810/3125   train_loss = 0.967\n",
      "2019-03-20T15:52:05.960040: Epoch   2 Batch 1830/3125   train_loss = 1.025\n",
      "2019-03-20T15:52:06.597894: Epoch   2 Batch 1850/3125   train_loss = 0.929\n",
      "2019-03-20T15:52:07.263028: Epoch   2 Batch 1870/3125   train_loss = 0.990\n",
      "2019-03-20T15:52:07.938083: Epoch   2 Batch 1890/3125   train_loss = 0.770\n",
      "2019-03-20T15:52:08.579409: Epoch   2 Batch 1910/3125   train_loss = 0.902\n",
      "2019-03-20T15:52:09.268848: Epoch   2 Batch 1930/3125   train_loss = 0.990\n",
      "2019-03-20T15:52:09.914639: Epoch   2 Batch 1950/3125   train_loss = 0.898\n",
      "2019-03-20T15:52:10.565389: Epoch   2 Batch 1970/3125   train_loss = 0.956\n",
      "2019-03-20T15:52:11.185388: Epoch   2 Batch 1990/3125   train_loss = 0.879\n",
      "2019-03-20T15:52:11.820267: Epoch   2 Batch 2010/3125   train_loss = 0.806\n",
      "2019-03-20T15:52:12.440265: Epoch   2 Batch 2030/3125   train_loss = 0.952\n",
      "2019-03-20T15:52:13.104407: Epoch   2 Batch 2050/3125   train_loss = 1.041\n",
      "2019-03-20T15:52:13.779957: Epoch   2 Batch 2070/3125   train_loss = 0.939\n",
      "2019-03-20T15:52:14.411365: Epoch   2 Batch 2090/3125   train_loss = 0.870\n",
      "2019-03-20T15:52:15.075506: Epoch   2 Batch 2110/3125   train_loss = 1.062\n",
      "2019-03-20T15:52:15.723281: Epoch   2 Batch 2130/3125   train_loss = 0.903\n",
      "2019-03-20T15:52:16.371552: Epoch   2 Batch 2150/3125   train_loss = 0.994\n",
      "2019-03-20T15:52:17.078847: Epoch   2 Batch 2170/3125   train_loss = 0.886\n",
      "2019-03-20T15:52:17.872941: Epoch   2 Batch 2190/3125   train_loss = 0.971\n",
      "2019-03-20T15:52:18.605034: Epoch   2 Batch 2210/3125   train_loss = 0.908\n",
      "2019-03-20T15:52:19.525609: Epoch   2 Batch 2230/3125   train_loss = 0.838\n",
      "2019-03-20T15:52:20.169415: Epoch   2 Batch 2250/3125   train_loss = 1.028\n",
      "2019-03-20T15:52:20.777013: Epoch   2 Batch 2270/3125   train_loss = 0.986\n",
      "2019-03-20T15:52:21.416356: Epoch   2 Batch 2290/3125   train_loss = 0.809\n",
      "2019-03-20T15:52:22.078513: Epoch   2 Batch 2310/3125   train_loss = 0.894\n",
      "2019-03-20T15:52:22.712402: Epoch   2 Batch 2330/3125   train_loss = 1.075\n",
      "2019-03-20T15:52:23.351744: Epoch   2 Batch 2350/3125   train_loss = 0.968\n",
      "2019-03-20T15:52:23.986623: Epoch   2 Batch 2370/3125   train_loss = 0.884\n",
      "2019-03-20T15:52:24.607612: Epoch   2 Batch 2390/3125   train_loss = 1.042\n",
      "2019-03-20T15:52:25.240507: Epoch   2 Batch 2410/3125   train_loss = 1.034\n",
      "2019-03-20T15:52:25.884809: Epoch   2 Batch 2430/3125   train_loss = 0.931\n",
      "2019-03-20T15:52:26.540024: Epoch   2 Batch 2450/3125   train_loss = 0.975\n",
      "2019-03-20T15:52:27.168951: Epoch   2 Batch 2470/3125   train_loss = 0.952\n",
      "2019-03-20T15:52:27.800358: Epoch   2 Batch 2490/3125   train_loss = 1.020\n",
      "2019-03-20T15:52:28.418372: Epoch   2 Batch 2510/3125   train_loss = 1.116\n",
      "2019-03-20T15:52:29.028450: Epoch   2 Batch 2530/3125   train_loss = 0.827\n",
      "2019-03-20T15:52:29.657377: Epoch   2 Batch 2550/3125   train_loss = 1.053\n",
      "2019-03-20T15:52:30.276879: Epoch   2 Batch 2570/3125   train_loss = 1.012\n",
      "2019-03-20T15:52:30.873566: Epoch   2 Batch 2590/3125   train_loss = 0.985\n",
      "2019-03-20T15:52:31.471245: Epoch   2 Batch 2610/3125   train_loss = 1.093\n",
      "2019-03-20T15:52:32.078843: Epoch   2 Batch 2630/3125   train_loss = 0.643\n",
      "2019-03-20T15:52:32.677513: Epoch   2 Batch 2650/3125   train_loss = 0.937\n",
      "2019-03-20T15:52:33.328761: Epoch   2 Batch 2670/3125   train_loss = 1.049\n",
      "2019-03-20T15:52:33.949256: Epoch   2 Batch 2690/3125   train_loss = 1.009\n",
      "2019-03-20T15:52:34.558342: Epoch   2 Batch 2710/3125   train_loss = 0.862\n",
      "2019-03-20T15:52:35.257699: Epoch   2 Batch 2730/3125   train_loss = 1.080\n",
      "2019-03-20T15:52:35.956067: Epoch   2 Batch 2750/3125   train_loss = 1.016\n",
      "2019-03-20T15:52:36.584992: Epoch   2 Batch 2770/3125   train_loss = 0.877\n",
      "2019-03-20T15:52:37.209951: Epoch   2 Batch 2790/3125   train_loss = 0.935\n",
      "2019-03-20T15:52:37.834909: Epoch   2 Batch 2810/3125   train_loss = 1.001\n",
      "2019-03-20T15:52:38.450445: Epoch   2 Batch 2830/3125   train_loss = 0.880\n",
      "2019-03-20T15:52:39.089787: Epoch   2 Batch 2850/3125   train_loss = 0.998\n",
      "2019-03-20T15:52:39.709785: Epoch   2 Batch 2870/3125   train_loss = 0.835\n",
      "2019-03-20T15:52:40.334744: Epoch   2 Batch 2890/3125   train_loss = 0.752\n",
      "2019-03-20T15:52:40.946311: Epoch   2 Batch 2910/3125   train_loss = 0.944\n",
      "2019-03-20T15:52:41.555397: Epoch   2 Batch 2930/3125   train_loss = 0.845\n",
      "2019-03-20T15:52:42.167956: Epoch   2 Batch 2950/3125   train_loss = 1.078\n",
      "2019-03-20T15:52:42.757202: Epoch   2 Batch 2970/3125   train_loss = 0.908\n",
      "2019-03-20T15:52:43.398529: Epoch   2 Batch 2990/3125   train_loss = 0.907\n",
      "2019-03-20T15:52:44.076063: Epoch   2 Batch 3010/3125   train_loss = 0.930\n",
      "2019-03-20T15:52:44.698046: Epoch   2 Batch 3030/3125   train_loss = 0.927\n",
      "2019-03-20T15:52:45.312588: Epoch   2 Batch 3050/3125   train_loss = 0.951\n",
      "2019-03-20T15:52:45.938043: Epoch   2 Batch 3070/3125   train_loss = 0.920\n",
      "2019-03-20T15:52:46.544649: Epoch   2 Batch 3090/3125   train_loss = 0.760\n",
      "2019-03-20T15:52:47.192424: Epoch   2 Batch 3110/3125   train_loss = 0.821\n",
      "2019-03-20T15:52:47.867485: Epoch   2 Batch   18/781   test_loss = 0.822\n",
      "2019-03-20T15:52:48.068359: Epoch   2 Batch   38/781   test_loss = 0.930\n",
      "2019-03-20T15:52:48.259814: Epoch   2 Batch   58/781   test_loss = 0.867\n",
      "2019-03-20T15:52:48.444326: Epoch   2 Batch   78/781   test_loss = 0.895\n",
      "2019-03-20T15:52:48.646198: Epoch   2 Batch   98/781   test_loss = 0.963\n",
      "2019-03-20T15:52:48.841124: Epoch   2 Batch  118/781   test_loss = 0.877\n",
      "2019-03-20T15:52:49.027124: Epoch   2 Batch  138/781   test_loss = 1.013\n",
      "2019-03-20T15:52:49.209157: Epoch   2 Batch  158/781   test_loss = 0.864\n",
      "2019-03-20T15:52:49.384243: Epoch   2 Batch  178/781   test_loss = 0.826\n",
      "2019-03-20T15:52:49.562804: Epoch   2 Batch  198/781   test_loss = 0.957\n",
      "2019-03-20T15:52:49.742851: Epoch   2 Batch  218/781   test_loss = 1.084\n",
      "2019-03-20T15:52:49.923395: Epoch   2 Batch  238/781   test_loss = 0.947\n",
      "2019-03-20T15:52:50.102449: Epoch   2 Batch  258/781   test_loss = 0.994\n",
      "2019-03-20T15:52:50.278034: Epoch   2 Batch  278/781   test_loss = 1.089\n",
      "2019-03-20T15:52:50.450145: Epoch   2 Batch  298/781   test_loss = 0.884\n",
      "2019-03-20T15:52:50.646067: Epoch   2 Batch  318/781   test_loss = 0.904\n",
      "2019-03-20T15:52:50.833551: Epoch   2 Batch  338/781   test_loss = 0.905\n",
      "2019-03-20T15:52:51.008639: Epoch   2 Batch  358/781   test_loss = 0.914\n",
      "2019-03-20T15:52:51.189184: Epoch   2 Batch  378/781   test_loss = 0.881\n",
      "2019-03-20T15:52:51.365264: Epoch   2 Batch  398/781   test_loss = 0.813\n",
      "2019-03-20T15:52:51.557214: Epoch   2 Batch  418/781   test_loss = 1.017\n",
      "2019-03-20T15:52:51.735774: Epoch   2 Batch  438/781   test_loss = 1.019\n",
      "2019-03-20T15:52:51.920781: Epoch   2 Batch  458/781   test_loss = 0.878\n",
      "2019-03-20T15:52:52.112238: Epoch   2 Batch  478/781   test_loss = 0.977\n",
      "2019-03-20T15:52:52.284844: Epoch   2 Batch  498/781   test_loss = 0.791\n",
      "2019-03-20T15:52:52.466876: Epoch   2 Batch  518/781   test_loss = 0.971\n",
      "2019-03-20T15:52:52.686604: Epoch   2 Batch  538/781   test_loss = 0.782\n",
      "2019-03-20T15:52:52.872109: Epoch   2 Batch  558/781   test_loss = 0.871\n",
      "2019-03-20T15:52:53.060114: Epoch   2 Batch  578/781   test_loss = 0.887\n",
      "2019-03-20T15:52:53.236666: Epoch   2 Batch  598/781   test_loss = 1.055\n",
      "2019-03-20T15:52:53.432586: Epoch   2 Batch  618/781   test_loss = 0.842\n",
      "2019-03-20T15:52:53.637434: Epoch   2 Batch  638/781   test_loss = 0.877\n",
      "2019-03-20T15:52:53.840297: Epoch   2 Batch  658/781   test_loss = 1.095\n",
      "2019-03-20T15:52:54.019848: Epoch   2 Batch  678/781   test_loss = 0.931\n",
      "2019-03-20T15:52:54.194440: Epoch   2 Batch  698/781   test_loss = 0.907\n",
      "2019-03-20T15:52:54.364567: Epoch   2 Batch  718/781   test_loss = 1.023\n",
      "2019-03-20T15:52:54.538167: Epoch   2 Batch  738/781   test_loss = 0.856\n",
      "2019-03-20T15:52:54.714246: Epoch   2 Batch  758/781   test_loss = 0.936\n",
      "2019-03-20T15:52:54.901734: Epoch   2 Batch  778/781   test_loss = 0.960\n",
      "2019-03-20T15:52:55.582245: Epoch   3 Batch    5/3125   train_loss = 0.974\n",
      "2019-03-20T15:52:56.179428: Epoch   3 Batch   25/3125   train_loss = 0.974\n",
      "2019-03-20T15:52:56.818770: Epoch   3 Batch   45/3125   train_loss = 0.891\n",
      "2019-03-20T15:52:57.448192: Epoch   3 Batch   65/3125   train_loss = 0.942\n",
      "2019-03-20T15:52:58.075633: Epoch   3 Batch   85/3125   train_loss = 0.863\n",
      "2019-03-20T15:52:58.705550: Epoch   3 Batch  105/3125   train_loss = 0.795\n",
      "2019-03-20T15:52:59.354813: Epoch   3 Batch  125/3125   train_loss = 0.967\n",
      "2019-03-20T15:53:00.044746: Epoch   3 Batch  145/3125   train_loss = 0.959\n",
      "2019-03-20T15:53:00.701945: Epoch   3 Batch  165/3125   train_loss = 0.918\n",
      "2019-03-20T15:53:01.363608: Epoch   3 Batch  185/3125   train_loss = 0.894\n",
      "2019-03-20T15:53:02.014854: Epoch   3 Batch  205/3125   train_loss = 0.846\n",
      "2019-03-20T15:53:02.689909: Epoch   3 Batch  225/3125   train_loss = 0.764\n",
      "2019-03-20T15:53:03.393235: Epoch   3 Batch  245/3125   train_loss = 1.078\n",
      "2019-03-20T15:53:04.054898: Epoch   3 Batch  265/3125   train_loss = 0.877\n",
      "2019-03-20T15:53:04.686303: Epoch   3 Batch  285/3125   train_loss = 0.912\n",
      "2019-03-20T15:53:05.350943: Epoch   3 Batch  305/3125   train_loss = 0.877\n",
      "2019-03-20T15:53:06.006159: Epoch   3 Batch  325/3125   train_loss = 1.013\n",
      "2019-03-20T15:53:06.655419: Epoch   3 Batch  345/3125   train_loss = 1.026\n",
      "2019-03-20T15:53:07.282858: Epoch   3 Batch  365/3125   train_loss = 0.881\n",
      "2019-03-20T15:53:07.937080: Epoch   3 Batch  385/3125   train_loss = 0.901\n",
      "2019-03-20T15:53:08.581878: Epoch   3 Batch  405/3125   train_loss = 0.910\n",
      "2019-03-20T15:53:09.249493: Epoch   3 Batch  425/3125   train_loss = 0.978\n",
      "2019-03-20T15:53:09.877427: Epoch   3 Batch  445/3125   train_loss = 0.940\n",
      "2019-03-20T15:53:10.512306: Epoch   3 Batch  465/3125   train_loss = 0.920\n",
      "2019-03-20T15:53:11.165537: Epoch   3 Batch  485/3125   train_loss = 1.014\n",
      "2019-03-20T15:53:11.847039: Epoch   3 Batch  505/3125   train_loss = 0.871\n",
      "2019-03-20T15:53:12.469021: Epoch   3 Batch  525/3125   train_loss = 0.994\n",
      "2019-03-20T15:53:13.115805: Epoch   3 Batch  545/3125   train_loss = 0.903\n",
      "2019-03-20T15:53:13.704555: Epoch   3 Batch  565/3125   train_loss = 1.104\n",
      "2019-03-20T15:53:14.276442: Epoch   3 Batch  585/3125   train_loss = 0.856\n",
      "2019-03-20T15:53:14.849320: Epoch   3 Batch  605/3125   train_loss = 0.873\n",
      "2019-03-20T15:53:15.412775: Epoch   3 Batch  625/3125   train_loss = 0.886\n",
      "2019-03-20T15:53:15.971766: Epoch   3 Batch  645/3125   train_loss = 0.961\n",
      "2019-03-20T15:53:16.544644: Epoch   3 Batch  665/3125   train_loss = 1.001\n",
      "2019-03-20T15:53:17.123474: Epoch   3 Batch  685/3125   train_loss = 0.851\n",
      "2019-03-20T15:53:17.706769: Epoch   3 Batch  705/3125   train_loss = 1.048\n",
      "2019-03-20T15:53:18.293536: Epoch   3 Batch  725/3125   train_loss = 0.907\n",
      "2019-03-20T15:53:18.873359: Epoch   3 Batch  745/3125   train_loss = 0.871\n",
      "2019-03-20T15:53:19.444254: Epoch   3 Batch  765/3125   train_loss = 0.898\n",
      "2019-03-20T15:53:20.037468: Epoch   3 Batch  785/3125   train_loss = 1.067\n",
      "2019-03-20T15:53:20.608859: Epoch   3 Batch  805/3125   train_loss = 0.867\n",
      "2019-03-20T15:53:21.172809: Epoch   3 Batch  825/3125   train_loss = 0.982\n",
      "2019-03-20T15:53:21.742713: Epoch   3 Batch  845/3125   train_loss = 0.908\n",
      "2019-03-20T15:53:22.317079: Epoch   3 Batch  865/3125   train_loss = 0.960\n",
      "2019-03-20T15:53:22.898886: Epoch   3 Batch  885/3125   train_loss = 0.933\n",
      "2019-03-20T15:53:23.484164: Epoch   3 Batch  905/3125   train_loss = 1.038\n",
      "2019-03-20T15:53:24.064979: Epoch   3 Batch  925/3125   train_loss = 0.906\n",
      "2019-03-20T15:53:24.644802: Epoch   3 Batch  945/3125   train_loss = 0.973\n",
      "2019-03-20T15:53:25.240000: Epoch   3 Batch  965/3125   train_loss = 0.800\n",
      "2019-03-20T15:53:25.811887: Epoch   3 Batch  985/3125   train_loss = 0.977\n",
      "2019-03-20T15:53:26.395182: Epoch   3 Batch 1005/3125   train_loss = 0.796\n",
      "2019-03-20T15:53:26.984427: Epoch   3 Batch 1025/3125   train_loss = 0.912\n",
      "2019-03-20T15:53:27.595003: Epoch   3 Batch 1045/3125   train_loss = 1.103\n",
      "2019-03-20T15:53:28.184249: Epoch   3 Batch 1065/3125   train_loss = 0.907\n",
      "2019-03-20T15:53:28.766552: Epoch   3 Batch 1085/3125   train_loss = 0.830\n",
      "2019-03-20T15:53:29.354807: Epoch   3 Batch 1105/3125   train_loss = 0.822\n",
      "2019-03-20T15:53:29.936613: Epoch   3 Batch 1125/3125   train_loss = 0.861\n",
      "2019-03-20T15:53:30.534292: Epoch   3 Batch 1145/3125   train_loss = 0.876\n",
      "2019-03-20T15:53:31.139862: Epoch   3 Batch 1165/3125   train_loss = 0.960\n",
      "2019-03-20T15:53:31.726630: Epoch   3 Batch 1185/3125   train_loss = 0.868\n",
      "2019-03-20T15:53:32.336707: Epoch   3 Batch 1205/3125   train_loss = 0.843\n",
      "2019-03-20T15:53:32.947282: Epoch   3 Batch 1225/3125   train_loss = 1.017\n",
      "2019-03-20T15:53:33.540497: Epoch   3 Batch 1245/3125   train_loss = 1.047\n",
      "2019-03-20T15:53:34.135200: Epoch   3 Batch 1265/3125   train_loss = 0.903\n",
      "2019-03-20T15:53:34.727422: Epoch   3 Batch 1285/3125   train_loss = 0.927\n",
      "2019-03-20T15:53:35.315181: Epoch   3 Batch 1305/3125   train_loss = 0.816\n",
      "2019-03-20T15:53:35.883595: Epoch   3 Batch 1325/3125   train_loss = 0.890\n",
      "2019-03-20T15:53:36.463418: Epoch   3 Batch 1345/3125   train_loss = 0.909\n",
      "2019-03-20T15:53:37.036296: Epoch   3 Batch 1365/3125   train_loss = 0.802\n",
      "2019-03-20T15:53:37.623063: Epoch   3 Batch 1385/3125   train_loss = 0.800\n",
      "2019-03-20T15:53:38.213302: Epoch   3 Batch 1405/3125   train_loss = 0.919\n",
      "2019-03-20T15:53:38.818916: Epoch   3 Batch 1425/3125   train_loss = 1.083\n",
      "2019-03-20T15:53:39.413619: Epoch   3 Batch 1445/3125   train_loss = 1.019\n",
      "2019-03-20T15:53:40.005346: Epoch   3 Batch 1465/3125   train_loss = 0.902\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:53:40.582193: Epoch   3 Batch 1485/3125   train_loss = 0.989\n",
      "2019-03-20T15:53:41.157551: Epoch   3 Batch 1505/3125   train_loss = 0.777\n",
      "2019-03-20T15:53:41.744318: Epoch   3 Batch 1525/3125   train_loss = 0.815\n",
      "2019-03-20T15:53:42.333564: Epoch   3 Batch 1545/3125   train_loss = 0.928\n",
      "2019-03-20T15:53:42.911898: Epoch   3 Batch 1565/3125   train_loss = 0.993\n",
      "2019-03-20T15:53:43.498666: Epoch   3 Batch 1585/3125   train_loss = 0.824\n",
      "2019-03-20T15:53:44.085928: Epoch   3 Batch 1605/3125   train_loss = 0.873\n",
      "2019-03-20T15:53:44.671703: Epoch   3 Batch 1625/3125   train_loss = 1.002\n",
      "2019-03-20T15:53:45.267894: Epoch   3 Batch 1645/3125   train_loss = 1.027\n",
      "2019-03-20T15:53:45.879460: Epoch   3 Batch 1665/3125   train_loss = 0.981\n",
      "2019-03-20T15:53:46.554018: Epoch   3 Batch 1685/3125   train_loss = 1.003\n",
      "2019-03-20T15:53:47.181953: Epoch   3 Batch 1705/3125   train_loss = 0.921\n",
      "2019-03-20T15:53:47.772192: Epoch   3 Batch 1725/3125   train_loss = 0.893\n",
      "2019-03-20T15:53:48.401614: Epoch   3 Batch 1745/3125   train_loss = 0.826\n",
      "2019-03-20T15:53:49.018637: Epoch   3 Batch 1765/3125   train_loss = 0.904\n",
      "2019-03-20T15:53:49.653019: Epoch   3 Batch 1785/3125   train_loss = 0.988\n",
      "2019-03-20T15:53:50.278970: Epoch   3 Batch 1805/3125   train_loss = 0.984\n",
      "2019-03-20T15:53:50.862761: Epoch   3 Batch 1825/3125   train_loss = 1.017\n",
      "2019-03-20T15:53:51.457464: Epoch   3 Batch 1845/3125   train_loss = 0.953\n",
      "2019-03-20T15:53:52.048198: Epoch   3 Batch 1865/3125   train_loss = 0.812\n",
      "2019-03-20T15:53:52.699940: Epoch   3 Batch 1885/3125   train_loss = 0.981\n",
      "2019-03-20T15:53:53.336306: Epoch   3 Batch 1905/3125   train_loss = 0.869\n",
      "2019-03-20T15:53:54.006897: Epoch   3 Batch 1925/3125   train_loss = 0.850\n",
      "2019-03-20T15:53:54.664097: Epoch   3 Batch 1945/3125   train_loss = 0.885\n",
      "2019-03-20T15:53:55.270702: Epoch   3 Batch 1965/3125   train_loss = 0.810\n",
      "2019-03-20T15:53:55.862429: Epoch   3 Batch 1985/3125   train_loss = 0.816\n",
      "2019-03-20T15:53:56.445228: Epoch   3 Batch 2005/3125   train_loss = 0.953\n",
      "2019-03-20T15:53:57.089531: Epoch   3 Batch 2025/3125   train_loss = 0.918\n",
      "2019-03-20T15:53:57.677785: Epoch   3 Batch 2045/3125   train_loss = 0.757\n",
      "2019-03-20T15:53:58.306711: Epoch   3 Batch 2065/3125   train_loss = 0.741\n",
      "2019-03-20T15:53:58.930679: Epoch   3 Batch 2085/3125   train_loss = 1.000\n",
      "2019-03-20T15:53:59.517941: Epoch   3 Batch 2105/3125   train_loss = 0.873\n",
      "2019-03-20T15:54:00.099748: Epoch   3 Batch 2125/3125   train_loss = 0.953\n",
      "2019-03-20T15:54:00.689490: Epoch   3 Batch 2145/3125   train_loss = 1.055\n",
      "2019-03-20T15:54:01.287168: Epoch   3 Batch 2165/3125   train_loss = 0.853\n",
      "2019-03-20T15:54:01.890304: Epoch   3 Batch 2185/3125   train_loss = 0.975\n",
      "2019-03-20T15:54:02.480046: Epoch   3 Batch 2205/3125   train_loss = 0.881\n",
      "2019-03-20T15:54:03.070285: Epoch   3 Batch 2225/3125   train_loss = 0.888\n",
      "2019-03-20T15:54:03.662508: Epoch   3 Batch 2245/3125   train_loss = 0.809\n",
      "2019-03-20T15:54:04.246793: Epoch   3 Batch 2265/3125   train_loss = 0.898\n",
      "2019-03-20T15:54:04.853401: Epoch   3 Batch 2285/3125   train_loss = 1.110\n",
      "2019-03-20T15:54:05.501175: Epoch   3 Batch 2305/3125   train_loss = 0.816\n",
      "2019-03-20T15:54:06.152918: Epoch   3 Batch 2325/3125   train_loss = 0.837\n",
      "2019-03-20T15:54:06.791764: Epoch   3 Batch 2345/3125   train_loss = 0.852\n",
      "2019-03-20T15:54:07.447970: Epoch   3 Batch 2365/3125   train_loss = 0.775\n",
      "2019-03-20T15:54:08.088304: Epoch   3 Batch 2385/3125   train_loss = 0.954\n",
      "2019-03-20T15:54:08.683008: Epoch   3 Batch 2405/3125   train_loss = 0.914\n",
      "2019-03-20T15:54:09.266303: Epoch   3 Batch 2425/3125   train_loss = 0.930\n",
      "2019-03-20T15:54:09.887789: Epoch   3 Batch 2445/3125   train_loss = 0.947\n",
      "2019-03-20T15:54:10.478523: Epoch   3 Batch 2465/3125   train_loss = 0.757\n",
      "2019-03-20T15:54:11.065785: Epoch   3 Batch 2485/3125   train_loss = 0.793\n",
      "2019-03-20T15:54:11.655528: Epoch   3 Batch 2505/3125   train_loss = 0.843\n",
      "2019-03-20T15:54:12.245271: Epoch   3 Batch 2525/3125   train_loss = 0.847\n",
      "2019-03-20T15:54:12.846918: Epoch   3 Batch 2545/3125   train_loss = 0.978\n",
      "2019-03-20T15:54:13.446084: Epoch   3 Batch 2565/3125   train_loss = 0.849\n",
      "2019-03-20T15:54:14.037811: Epoch   3 Batch 2585/3125   train_loss = 0.787\n",
      "2019-03-20T15:54:14.646401: Epoch   3 Batch 2605/3125   train_loss = 0.832\n",
      "2019-03-20T15:54:15.254993: Epoch   3 Batch 2625/3125   train_loss = 0.998\n",
      "2019-03-20T15:54:15.852174: Epoch   3 Batch 2645/3125   train_loss = 0.878\n",
      "2019-03-20T15:54:16.446877: Epoch   3 Batch 2665/3125   train_loss = 0.979\n",
      "2019-03-20T15:54:17.047532: Epoch   3 Batch 2685/3125   train_loss = 0.882\n",
      "2019-03-20T15:54:17.662075: Epoch   3 Batch 2705/3125   train_loss = 0.757\n",
      "2019-03-20T15:54:18.309850: Epoch   3 Batch 2725/3125   train_loss = 0.962\n",
      "2019-03-20T15:54:18.907032: Epoch   3 Batch 2745/3125   train_loss = 0.917\n",
      "2019-03-20T15:54:19.498263: Epoch   3 Batch 2765/3125   train_loss = 0.773\n",
      "2019-03-20T15:54:20.090981: Epoch   3 Batch 2785/3125   train_loss = 0.967\n",
      "2019-03-20T15:54:20.692628: Epoch   3 Batch 2805/3125   train_loss = 0.873\n",
      "2019-03-20T15:54:21.296258: Epoch   3 Batch 2825/3125   train_loss = 0.864\n",
      "2019-03-20T15:54:21.890465: Epoch   3 Batch 2845/3125   train_loss = 0.887\n",
      "2019-03-20T15:54:22.494592: Epoch   3 Batch 2865/3125   train_loss = 0.841\n",
      "2019-03-20T15:54:23.105662: Epoch   3 Batch 2885/3125   train_loss = 0.980\n",
      "2019-03-20T15:54:23.731614: Epoch   3 Batch 2905/3125   train_loss = 0.957\n",
      "2019-03-20T15:54:24.356571: Epoch   3 Batch 2925/3125   train_loss = 0.879\n",
      "2019-03-20T15:54:24.940362: Epoch   3 Batch 2945/3125   train_loss = 0.916\n",
      "2019-03-20T15:54:25.565817: Epoch   3 Batch 2965/3125   train_loss = 1.002\n",
      "2019-03-20T15:54:26.159030: Epoch   3 Batch 2985/3125   train_loss = 0.854\n",
      "2019-03-20T15:54:26.760182: Epoch   3 Batch 3005/3125   train_loss = 0.791\n",
      "2019-03-20T15:54:27.387124: Epoch   3 Batch 3025/3125   train_loss = 0.879\n",
      "2019-03-20T15:54:28.018035: Epoch   3 Batch 3045/3125   train_loss = 0.865\n",
      "2019-03-20T15:54:28.615218: Epoch   3 Batch 3065/3125   train_loss = 0.849\n",
      "2019-03-20T15:54:29.230257: Epoch   3 Batch 3085/3125   train_loss = 0.858\n",
      "2019-03-20T15:54:29.829919: Epoch   3 Batch 3105/3125   train_loss = 0.968\n",
      "2019-03-20T15:54:30.548126: Epoch   3 Batch   17/781   test_loss = 0.943\n",
      "2019-03-20T15:54:30.723212: Epoch   3 Batch   37/781   test_loss = 0.905\n",
      "2019-03-20T15:54:30.900286: Epoch   3 Batch   57/781   test_loss = 1.005\n",
      "2019-03-20T15:54:31.067932: Epoch   3 Batch   77/781   test_loss = 0.910\n",
      "2019-03-20T15:54:31.236076: Epoch   3 Batch   97/781   test_loss = 0.797\n",
      "2019-03-20T15:54:31.404716: Epoch   3 Batch  117/781   test_loss = 0.950\n",
      "2019-03-20T15:54:31.583274: Epoch   3 Batch  137/781   test_loss = 0.927\n",
      "2019-03-20T15:54:31.750426: Epoch   3 Batch  157/781   test_loss = 0.901\n",
      "2019-03-20T15:54:31.919065: Epoch   3 Batch  177/781   test_loss = 0.870\n",
      "2019-03-20T15:54:32.088698: Epoch   3 Batch  197/781   test_loss = 0.915\n",
      "2019-03-20T15:54:32.259322: Epoch   3 Batch  217/781   test_loss = 0.788\n",
      "2019-03-20T15:54:32.448297: Epoch   3 Batch  237/781   test_loss = 0.760\n",
      "2019-03-20T15:54:32.609992: Epoch   3 Batch  257/781   test_loss = 1.016\n",
      "2019-03-20T15:54:32.789048: Epoch   3 Batch  277/781   test_loss = 0.990\n",
      "2019-03-20T15:54:32.965127: Epoch   3 Batch  297/781   test_loss = 0.985\n",
      "2019-03-20T15:54:33.125335: Epoch   3 Batch  317/781   test_loss = 1.006\n",
      "2019-03-20T15:54:33.300422: Epoch   3 Batch  337/781   test_loss = 0.929\n",
      "2019-03-20T15:54:33.471543: Epoch   3 Batch  357/781   test_loss = 0.920\n",
      "2019-03-20T15:54:33.656550: Epoch   3 Batch  377/781   test_loss = 0.911\n",
      "2019-03-20T15:54:33.832135: Epoch   3 Batch  397/781   test_loss = 0.915\n",
      "2019-03-20T15:54:34.001765: Epoch   3 Batch  417/781   test_loss = 0.864\n",
      "2019-03-20T15:54:34.171397: Epoch   3 Batch  437/781   test_loss = 0.800\n",
      "2019-03-20T15:54:34.339541: Epoch   3 Batch  457/781   test_loss = 0.741\n",
      "2019-03-20T15:54:34.511652: Epoch   3 Batch  477/781   test_loss = 0.890\n",
      "2019-03-20T15:54:34.683765: Epoch   3 Batch  497/781   test_loss = 0.816\n",
      "2019-03-20T15:54:34.867284: Epoch   3 Batch  517/781   test_loss = 0.815\n",
      "2019-03-20T15:54:35.059732: Epoch   3 Batch  537/781   test_loss = 0.913\n",
      "2019-03-20T15:54:35.236306: Epoch   3 Batch  557/781   test_loss = 1.030\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:54:35.424787: Epoch   3 Batch  577/781   test_loss = 0.912\n",
      "2019-03-20T15:54:35.617730: Epoch   3 Batch  597/781   test_loss = 0.861\n",
      "2019-03-20T15:54:35.808691: Epoch   3 Batch  617/781   test_loss = 0.852\n",
      "2019-03-20T15:54:35.984769: Epoch   3 Batch  637/781   test_loss = 0.793\n",
      "2019-03-20T15:54:36.159909: Epoch   3 Batch  657/781   test_loss = 1.012\n",
      "2019-03-20T15:54:36.343375: Epoch   3 Batch  677/781   test_loss = 0.882\n",
      "2019-03-20T15:54:36.533839: Epoch   3 Batch  697/781   test_loss = 0.955\n",
      "2019-03-20T15:54:36.723807: Epoch   3 Batch  717/781   test_loss = 0.851\n",
      "2019-03-20T15:54:36.896416: Epoch   3 Batch  737/781   test_loss = 0.724\n",
      "2019-03-20T15:54:37.076461: Epoch   3 Batch  757/781   test_loss = 1.059\n",
      "2019-03-20T15:54:37.261469: Epoch   3 Batch  777/781   test_loss = 0.974\n",
      "2019-03-20T15:54:37.780321: Epoch   4 Batch    0/3125   train_loss = 1.009\n",
      "2019-03-20T15:54:38.313482: Epoch   4 Batch   20/3125   train_loss = 0.847\n",
      "2019-03-20T15:54:38.884378: Epoch   4 Batch   40/3125   train_loss = 0.901\n",
      "2019-03-20T15:54:39.461720: Epoch   4 Batch   60/3125   train_loss = 0.738\n",
      "2019-03-20T15:54:40.037079: Epoch   4 Batch   80/3125   train_loss = 0.865\n",
      "2019-03-20T15:54:40.600534: Epoch   4 Batch  100/3125   train_loss = 0.962\n",
      "2019-03-20T15:54:41.153572: Epoch   4 Batch  120/3125   train_loss = 0.970\n",
      "2019-03-20T15:54:41.776051: Epoch   4 Batch  140/3125   train_loss = 0.975\n",
      "2019-03-20T15:54:42.423330: Epoch   4 Batch  160/3125   train_loss = 0.812\n",
      "2019-03-20T15:54:43.009599: Epoch   4 Batch  180/3125   train_loss = 0.871\n",
      "2019-03-20T15:54:43.570575: Epoch   4 Batch  200/3125   train_loss = 1.104\n",
      "2019-03-20T15:54:44.199501: Epoch   4 Batch  220/3125   train_loss = 0.865\n",
      "2019-03-20T15:54:44.813052: Epoch   4 Batch  240/3125   train_loss = 0.970\n",
      "2019-03-20T15:54:45.464298: Epoch   4 Batch  260/3125   train_loss = 0.926\n",
      "2019-03-20T15:54:46.048585: Epoch   4 Batch  280/3125   train_loss = 0.940\n",
      "2019-03-20T15:54:46.623944: Epoch   4 Batch  300/3125   train_loss = 1.025\n",
      "2019-03-20T15:54:47.204759: Epoch   4 Batch  320/3125   train_loss = 0.977\n",
      "2019-03-20T15:54:47.805412: Epoch   4 Batch  340/3125   train_loss = 0.745\n",
      "2019-03-20T15:54:48.454675: Epoch   4 Batch  360/3125   train_loss = 0.868\n",
      "2019-03-20T15:54:49.070705: Epoch   4 Batch  380/3125   train_loss = 0.866\n",
      "2019-03-20T15:54:49.707073: Epoch   4 Batch  400/3125   train_loss = 0.816\n",
      "2019-03-20T15:54:50.312191: Epoch   4 Batch  420/3125   train_loss = 0.806\n",
      "2019-03-20T15:54:50.912846: Epoch   4 Batch  440/3125   train_loss = 0.917\n",
      "2019-03-20T15:54:51.519949: Epoch   4 Batch  460/3125   train_loss = 0.896\n",
      "2019-03-20T15:54:52.107707: Epoch   4 Batch  480/3125   train_loss = 0.992\n",
      "2019-03-20T15:54:52.691497: Epoch   4 Batch  500/3125   train_loss = 0.701\n",
      "2019-03-20T15:54:53.316457: Epoch   4 Batch  520/3125   train_loss = 0.883\n",
      "2019-03-20T15:54:53.955303: Epoch   4 Batch  540/3125   train_loss = 0.817\n",
      "2019-03-20T15:54:54.598614: Epoch   4 Batch  560/3125   train_loss = 1.041\n",
      "2019-03-20T15:54:55.239940: Epoch   4 Batch  580/3125   train_loss = 0.987\n",
      "2019-03-20T15:54:55.926898: Epoch   4 Batch  600/3125   train_loss = 0.916\n",
      "2019-03-20T15:54:56.560289: Epoch   4 Batch  620/3125   train_loss = 0.937\n",
      "2019-03-20T15:54:57.239312: Epoch   4 Batch  640/3125   train_loss = 0.876\n",
      "2019-03-20T15:54:57.914861: Epoch   4 Batch  660/3125   train_loss = 0.940\n",
      "2019-03-20T15:54:58.607276: Epoch   4 Batch  680/3125   train_loss = 0.944\n",
      "2019-03-20T15:54:59.320523: Epoch   4 Batch  700/3125   train_loss = 0.941\n",
      "2019-03-20T15:55:00.022361: Epoch   4 Batch  720/3125   train_loss = 0.789\n",
      "2019-03-20T15:55:00.676584: Epoch   4 Batch  740/3125   train_loss = 0.909\n",
      "2019-03-20T15:55:01.320886: Epoch   4 Batch  760/3125   train_loss = 0.777\n",
      "2019-03-20T15:55:01.974117: Epoch   4 Batch  780/3125   train_loss = 0.888\n",
      "2019-03-20T15:55:02.629331: Epoch   4 Batch  800/3125   train_loss = 0.789\n",
      "2019-03-20T15:55:03.298929: Epoch   4 Batch  820/3125   train_loss = 0.879\n",
      "2019-03-20T15:55:03.964063: Epoch   4 Batch  840/3125   train_loss = 0.826\n",
      "2019-03-20T15:55:04.648046: Epoch   4 Batch  860/3125   train_loss = 0.818\n",
      "2019-03-20T15:55:05.350877: Epoch   4 Batch  880/3125   train_loss = 0.802\n",
      "2019-03-20T15:55:05.996668: Epoch   4 Batch  900/3125   train_loss = 0.894\n",
      "2019-03-20T15:55:06.619642: Epoch   4 Batch  920/3125   train_loss = 0.933\n",
      "2019-03-20T15:55:07.255512: Epoch   4 Batch  940/3125   train_loss = 0.887\n",
      "2019-03-20T15:55:07.937511: Epoch   4 Batch  960/3125   train_loss = 0.915\n",
      "2019-03-20T15:55:08.601158: Epoch   4 Batch  980/3125   train_loss = 0.960\n",
      "2019-03-20T15:55:09.281172: Epoch   4 Batch 1000/3125   train_loss = 0.926\n",
      "2019-03-20T15:55:09.950770: Epoch   4 Batch 1020/3125   train_loss = 0.953\n",
      "2019-03-20T15:55:10.582177: Epoch   4 Batch 1040/3125   train_loss = 0.773\n",
      "2019-03-20T15:55:11.176382: Epoch   4 Batch 1060/3125   train_loss = 0.985\n",
      "2019-03-20T15:55:11.785966: Epoch   4 Batch 1080/3125   train_loss = 0.920\n",
      "2019-03-20T15:55:12.387116: Epoch   4 Batch 1100/3125   train_loss = 0.858\n",
      "2019-03-20T15:55:12.971402: Epoch   4 Batch 1120/3125   train_loss = 0.835\n",
      "2019-03-20T15:55:13.550730: Epoch   4 Batch 1140/3125   train_loss = 0.865\n",
      "2019-03-20T15:55:14.125096: Epoch   4 Batch 1160/3125   train_loss = 0.838\n",
      "2019-03-20T15:55:14.709382: Epoch   4 Batch 1180/3125   train_loss = 0.860\n",
      "2019-03-20T15:55:15.285238: Epoch   4 Batch 1200/3125   train_loss = 0.960\n",
      "2019-03-20T15:55:15.879444: Epoch   4 Batch 1220/3125   train_loss = 0.945\n",
      "2019-03-20T15:55:16.520276: Epoch   4 Batch 1240/3125   train_loss = 0.783\n",
      "2019-03-20T15:55:17.187888: Epoch   4 Batch 1260/3125   train_loss = 0.850\n",
      "2019-03-20T15:55:17.860959: Epoch   4 Batch 1280/3125   train_loss = 0.906\n",
      "2019-03-20T15:55:18.526590: Epoch   4 Batch 1300/3125   train_loss = 0.814\n",
      "2019-03-20T15:55:19.188252: Epoch   4 Batch 1320/3125   train_loss = 0.849\n",
      "2019-03-20T15:55:19.829101: Epoch   4 Batch 1340/3125   train_loss = 0.740\n",
      "2019-03-20T15:55:20.488762: Epoch   4 Batch 1360/3125   train_loss = 0.842\n",
      "2019-03-20T15:55:21.159353: Epoch   4 Batch 1380/3125   train_loss = 0.843\n",
      "2019-03-20T15:55:21.803159: Epoch   4 Batch 1400/3125   train_loss = 0.914\n",
      "2019-03-20T15:55:22.424149: Epoch   4 Batch 1420/3125   train_loss = 0.883\n",
      "2019-03-20T15:55:23.087300: Epoch   4 Batch 1440/3125   train_loss = 0.764\n",
      "2019-03-20T15:55:23.749956: Epoch   4 Batch 1460/3125   train_loss = 0.886\n",
      "2019-03-20T15:55:24.408144: Epoch   4 Batch 1480/3125   train_loss = 0.856\n",
      "2019-03-20T15:55:25.080226: Epoch   4 Batch 1500/3125   train_loss = 0.889\n",
      "2019-03-20T15:55:25.735437: Epoch   4 Batch 1520/3125   train_loss = 0.809\n",
      "2019-03-20T15:55:26.409499: Epoch   4 Batch 1540/3125   train_loss = 0.966\n",
      "2019-03-20T15:55:27.038923: Epoch   4 Batch 1560/3125   train_loss = 0.803\n",
      "2019-03-20T15:55:27.632633: Epoch   4 Batch 1580/3125   train_loss = 0.981\n",
      "2019-03-20T15:55:28.266023: Epoch   4 Batch 1600/3125   train_loss = 0.831\n",
      "2019-03-20T15:55:28.857750: Epoch   4 Batch 1620/3125   train_loss = 0.841\n",
      "2019-03-20T15:55:29.444517: Epoch   4 Batch 1640/3125   train_loss = 0.965\n",
      "2019-03-20T15:55:30.040211: Epoch   4 Batch 1660/3125   train_loss = 1.009\n",
      "2019-03-20T15:55:30.656910: Epoch   4 Batch 1680/3125   train_loss = 0.885\n",
      "2019-03-20T15:55:31.252604: Epoch   4 Batch 1700/3125   train_loss = 0.777\n",
      "2019-03-20T15:55:31.855243: Epoch   4 Batch 1720/3125   train_loss = 0.872\n",
      "2019-03-20T15:55:32.462842: Epoch   4 Batch 1740/3125   train_loss = 0.953\n",
      "2019-03-20T15:55:33.046136: Epoch   4 Batch 1760/3125   train_loss = 0.897\n",
      "2019-03-20T15:55:33.647783: Epoch   4 Batch 1780/3125   train_loss = 0.889\n",
      "2019-03-20T15:55:34.245958: Epoch   4 Batch 1800/3125   train_loss = 0.809\n",
      "2019-03-20T15:55:34.841653: Epoch   4 Batch 1820/3125   train_loss = 0.837\n",
      "2019-03-20T15:55:35.426931: Epoch   4 Batch 1840/3125   train_loss = 0.905\n",
      "2019-03-20T15:55:36.020146: Epoch   4 Batch 1860/3125   train_loss = 0.927\n",
      "2019-03-20T15:55:36.605424: Epoch   4 Batch 1880/3125   train_loss = 0.862\n",
      "2019-03-20T15:55:37.183263: Epoch   4 Batch 1900/3125   train_loss = 0.791\n",
      "2019-03-20T15:55:37.767549: Epoch   4 Batch 1920/3125   train_loss = 0.855\n",
      "2019-03-20T15:55:38.353821: Epoch   4 Batch 1940/3125   train_loss = 0.831\n",
      "2019-03-20T15:55:38.968364: Epoch   4 Batch 1960/3125   train_loss = 0.780\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-03-20T15:55:39.598281: Epoch   4 Batch 1980/3125   train_loss = 0.854\n",
      "2019-03-20T15:55:40.228696: Epoch   4 Batch 2000/3125   train_loss = 1.044\n",
      "2019-03-20T15:55:40.843238: Epoch   4 Batch 2020/3125   train_loss = 0.962\n",
      "2019-03-20T15:55:41.445381: Epoch   4 Batch 2040/3125   train_loss = 0.807\n",
      "2019-03-20T15:55:42.068852: Epoch   4 Batch 2060/3125   train_loss = 0.834\n",
      "2019-03-20T15:55:42.685874: Epoch   4 Batch 2080/3125   train_loss = 1.021\n",
      "2019-03-20T15:55:43.296944: Epoch   4 Batch 2100/3125   train_loss = 0.849\n",
      "2019-03-20T15:55:43.907023: Epoch   4 Batch 2120/3125   train_loss = 0.810\n",
      "2019-03-20T15:55:44.525534: Epoch   4 Batch 2140/3125   train_loss = 0.831\n",
      "2019-03-20T15:55:45.116268: Epoch   4 Batch 2160/3125   train_loss = 0.869\n",
      "2019-03-20T15:55:45.709979: Epoch   4 Batch 2180/3125   train_loss = 0.938\n",
      "2019-03-20T15:55:46.291290: Epoch   4 Batch 2200/3125   train_loss = 0.819\n",
      "2019-03-20T15:55:46.889464: Epoch   4 Batch 2220/3125   train_loss = 0.852\n",
      "2019-03-20T15:55:47.496567: Epoch   4 Batch 2240/3125   train_loss = 0.848\n",
      "2019-03-20T15:55:48.085317: Epoch   4 Batch 2260/3125   train_loss = 0.911\n",
      "2019-03-20T15:55:48.676548: Epoch   4 Batch 2280/3125   train_loss = 0.949\n",
      "2019-03-20T15:55:49.297538: Epoch   4 Batch 2300/3125   train_loss = 0.843\n",
      "2019-03-20T15:55:49.893730: Epoch   4 Batch 2320/3125   train_loss = 0.932\n",
      "2019-03-20T15:55:50.481984: Epoch   4 Batch 2340/3125   train_loss = 0.841\n",
      "2019-03-20T15:55:51.075199: Epoch   4 Batch 2360/3125   train_loss = 0.852\n",
      "2019-03-20T15:55:51.696190: Epoch   4 Batch 2380/3125   train_loss = 0.857\n",
      "2019-03-20T15:55:52.286924: Epoch   4 Batch 2400/3125   train_loss = 0.989\n",
      "2019-03-20T15:55:52.937179: Epoch   4 Batch 2420/3125   train_loss = 0.753\n",
      "2019-03-20T15:55:53.578505: Epoch   4 Batch 2440/3125   train_loss = 0.766\n",
      "2019-03-20T15:55:54.202968: Epoch   4 Batch 2460/3125   train_loss = 0.834\n",
      "2019-03-20T15:55:54.825446: Epoch   4 Batch 2480/3125   train_loss = 0.938\n",
      "2019-03-20T15:55:55.547124: Epoch   4 Batch 2500/3125   train_loss = 0.793\n",
      "2019-03-20T15:55:56.242516: Epoch   4 Batch 2520/3125   train_loss = 0.889\n",
      "2019-03-20T15:55:56.928977: Epoch   4 Batch 2540/3125   train_loss = 0.828\n",
      "2019-03-20T15:55:57.608003: Epoch   4 Batch 2560/3125   train_loss = 0.642\n",
      "2019-03-20T15:55:58.272638: Epoch   4 Batch 2580/3125   train_loss = 0.872\n",
      "2019-03-20T15:55:58.912477: Epoch   4 Batch 2600/3125   train_loss = 0.910\n",
      "2019-03-20T15:55:59.584059: Epoch   4 Batch 2620/3125   train_loss = 0.804\n",
      "2019-03-20T15:56:00.253658: Epoch   4 Batch 2640/3125   train_loss = 0.816\n",
      "2019-03-20T15:56:00.874648: Epoch   4 Batch 2660/3125   train_loss = 1.001\n",
      "2019-03-20T15:56:01.560120: Epoch   4 Batch 2680/3125   train_loss = 0.756\n",
      "2019-03-20T15:56:02.212357: Epoch   4 Batch 2700/3125   train_loss = 0.906\n",
      "2019-03-20T15:56:02.831860: Epoch   4 Batch 2720/3125   train_loss = 0.741\n",
      "2019-03-20T15:56:03.464754: Epoch   4 Batch 2740/3125   train_loss = 0.868\n",
      "2019-03-20T15:56:04.128896: Epoch   4 Batch 2760/3125   train_loss = 0.816\n",
      "2019-03-20T15:56:04.763279: Epoch   4 Batch 2780/3125   train_loss = 0.781\n",
      "2019-03-20T15:56:05.417006: Epoch   4 Batch 2800/3125   train_loss = 1.044\n",
      "2019-03-20T15:56:06.062300: Epoch   4 Batch 2820/3125   train_loss = 1.024\n",
      "2019-03-20T15:56:06.685275: Epoch   4 Batch 2840/3125   train_loss = 0.834\n",
      "2019-03-20T15:56:07.339498: Epoch   4 Batch 2860/3125   train_loss = 0.764\n",
      "2019-03-20T15:56:08.040344: Epoch   4 Batch 2880/3125   train_loss = 0.867\n",
      "2019-03-20T15:56:08.654390: Epoch   4 Batch 2900/3125   train_loss = 0.877\n",
      "2019-03-20T15:56:09.310101: Epoch   4 Batch 2920/3125   train_loss = 0.863\n",
      "2019-03-20T15:56:09.990116: Epoch   4 Batch 2940/3125   train_loss = 0.939\n",
      "2019-03-20T15:56:10.623505: Epoch   4 Batch 2960/3125   train_loss = 0.911\n",
      "2019-03-20T15:56:11.288144: Epoch   4 Batch 2980/3125   train_loss = 0.818\n",
      "2019-03-20T15:56:11.978079: Epoch   4 Batch 3000/3125   train_loss = 0.969\n",
      "2019-03-20T15:56:12.628828: Epoch   4 Batch 3020/3125   train_loss = 0.996\n",
      "2019-03-20T15:56:13.302395: Epoch   4 Batch 3040/3125   train_loss = 0.900\n",
      "2019-03-20T15:56:13.979930: Epoch   4 Batch 3060/3125   train_loss = 0.815\n",
      "2019-03-20T15:56:14.638616: Epoch   4 Batch 3080/3125   train_loss = 1.037\n",
      "2019-03-20T15:56:15.310199: Epoch   4 Batch 3100/3125   train_loss = 1.019\n",
      "2019-03-20T15:56:15.978309: Epoch   4 Batch 3120/3125   train_loss = 0.826\n",
      "2019-03-20T15:56:16.274423: Epoch   4 Batch   16/781   test_loss = 0.857\n",
      "2019-03-20T15:56:16.455958: Epoch   4 Batch   36/781   test_loss = 0.937\n",
      "2019-03-20T15:56:16.661300: Epoch   4 Batch   56/781   test_loss = 0.914\n",
      "2019-03-20T15:56:16.855235: Epoch   4 Batch   76/781   test_loss = 0.994\n",
      "2019-03-20T15:56:17.051651: Epoch   4 Batch   96/781   test_loss = 0.995\n",
      "2019-03-20T15:56:17.239139: Epoch   4 Batch  116/781   test_loss = 0.839\n",
      "2019-03-20T15:56:17.430099: Epoch   4 Batch  136/781   test_loss = 0.828\n",
      "2019-03-20T15:56:17.627505: Epoch   4 Batch  156/781   test_loss = 0.871\n",
      "2019-03-20T15:56:17.823921: Epoch   4 Batch  176/781   test_loss = 0.872\n",
      "2019-03-20T15:56:18.033728: Epoch   4 Batch  196/781   test_loss = 0.782\n",
      "2019-03-20T15:56:18.231137: Epoch   4 Batch  216/781   test_loss = 1.003\n",
      "2019-03-20T15:56:18.429041: Epoch   4 Batch  236/781   test_loss = 0.846\n",
      "2019-03-20T15:56:18.623968: Epoch   4 Batch  256/781   test_loss = 0.833\n",
      "2019-03-20T15:56:18.832286: Epoch   4 Batch  276/781   test_loss = 1.090\n",
      "2019-03-20T15:56:19.051023: Epoch   4 Batch  296/781   test_loss = 0.868\n",
      "2019-03-20T15:56:19.267773: Epoch   4 Batch  316/781   test_loss = 0.846\n",
      "2019-03-20T15:56:19.470141: Epoch   4 Batch  336/781   test_loss = 0.768\n",
      "2019-03-20T15:56:19.692348: Epoch   4 Batch  356/781   test_loss = 0.857\n",
      "2019-03-20T15:56:19.921997: Epoch   4 Batch  376/781   test_loss = 0.938\n",
      "2019-03-20T15:56:20.120894: Epoch   4 Batch  396/781   test_loss = 0.868\n",
      "2019-03-20T15:56:20.311851: Epoch   4 Batch  416/781   test_loss = 0.954\n",
      "2019-03-20T15:56:20.500829: Epoch   4 Batch  436/781   test_loss = 0.901\n",
      "2019-03-20T15:56:20.689802: Epoch   4 Batch  456/781   test_loss = 0.732\n",
      "2019-03-20T15:56:20.885722: Epoch   4 Batch  476/781   test_loss = 0.929\n",
      "2019-03-20T15:56:21.076186: Epoch   4 Batch  496/781   test_loss = 1.019\n",
      "2019-03-20T15:56:21.263177: Epoch   4 Batch  516/781   test_loss = 0.760\n",
      "2019-03-20T15:56:21.452648: Epoch   4 Batch  536/781   test_loss = 0.993\n",
      "2019-03-20T15:56:21.676345: Epoch   4 Batch  556/781   test_loss = 0.818\n",
      "2019-03-20T15:56:21.871768: Epoch   4 Batch  576/781   test_loss = 0.939\n",
      "2019-03-20T15:56:22.065207: Epoch   4 Batch  596/781   test_loss = 1.002\n",
      "2019-03-20T15:56:22.274022: Epoch   4 Batch  616/781   test_loss = 0.957\n",
      "2019-03-20T15:56:22.474406: Epoch   4 Batch  636/781   test_loss = 0.833\n",
      "2019-03-20T15:56:22.671318: Epoch   4 Batch  656/781   test_loss = 0.866\n",
      "2019-03-20T15:56:22.883605: Epoch   4 Batch  676/781   test_loss = 1.082\n",
      "2019-03-20T15:56:23.088453: Epoch   4 Batch  696/781   test_loss = 0.882\n",
      "2019-03-20T15:56:23.281894: Epoch   4 Batch  716/781   test_loss = 0.897\n",
      "2019-03-20T15:56:23.478308: Epoch   4 Batch  736/781   test_loss = 1.055\n",
      "2019-03-20T15:56:23.664308: Epoch   4 Batch  756/781   test_loss = 0.820\n",
      "2019-03-20T15:56:23.851301: Epoch   4 Batch  776/781   test_loss = 0.771\n",
      "Model Trained and Saved\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "losses = {'train':[], 'test':[]}\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    \n",
    "    #搜集数据给tensorBoard用\n",
    "    # Keep track of gradient values and sparsity\n",
    "    grad_summaries = []\n",
    "    for g, v in gradients:\n",
    "        if g is not None:\n",
    "            grad_hist_summary = tf.summary.histogram(\"{}/grad/hist\".format(v.name.replace(':', '_')), g)\n",
    "            sparsity_summary = tf.summary.scalar(\"{}/grad/sparsity\".format(v.name.replace(':', '_')), tf.nn.zero_fraction(g))\n",
    "            grad_summaries.append(grad_hist_summary)\n",
    "            grad_summaries.append(sparsity_summary)\n",
    "    grad_summaries_merged = tf.summary.merge(grad_summaries)\n",
    "        \n",
    "    # Output directory for models and summaries\n",
    "    timestamp = str(int(time.time()))\n",
    "    #out_dir = os.path.abspath(os.path.join(os.path.curdir, \"runs\", timestamp))\n",
    "    out_dir = = \"E:/jupyter-src/Movie_Recommender_data/\"\n",
    "    print(\"Writing to {}\\n\".format(out_dir))\n",
    "     \n",
    "    # Summaries for loss and accuracy\n",
    "    loss_summary = tf.summary.scalar(\"loss\", loss)\n",
    "\n",
    "    # Train Summaries\n",
    "    train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])\n",
    "    train_summary_dir = os.path.join(out_dir, \"summaries\", \"train\")\n",
    "    train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)\n",
    "\n",
    "    # Inference summaries\n",
    "    inference_summary_op = tf.summary.merge([loss_summary])\n",
    "    inference_summary_dir = os.path.join(out_dir, \"summaries\", \"inference\")\n",
    "    inference_summary_writer = tf.summary.FileWriter(inference_summary_dir, sess.graph)\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver = tf.train.Saver()\n",
    "    for epoch_i in range(num_epochs):\n",
    "        \n",
    "        #将数据集分成训练集和测试集，随机种子不固定\n",
    "        train_X,test_X, train_y, test_y = train_test_split(features,  \n",
    "                                                           targets_values,  \n",
    "                                                           test_size = 0.2,  \n",
    "                                                           random_state = 0)  \n",
    "        \n",
    "        train_batches = get_batches(train_X, train_y, batch_size)\n",
    "        test_batches = get_batches(test_X, test_y, batch_size)\n",
    "    \n",
    "        #训练的迭代，保存训练损失\n",
    "        for batch_i in range(len(train_X) // batch_size):\n",
    "            x, y = next(train_batches)\n",
    "\n",
    "            categories = np.zeros([batch_size, 18])\n",
    "            for i in range(batch_size):\n",
    "                categories[i] = x.take(6,1)[i]\n",
    "\n",
    "            titles = np.zeros([batch_size, sentences_size])\n",
    "            for i in range(batch_size):\n",
    "                titles[i] = x.take(5,1)[i]\n",
    "\n",
    "            feed = {\n",
    "                uid: np.reshape(x.take(0,1), [batch_size, 1]),\n",
    "                user_gender: np.reshape(x.take(2,1), [batch_size, 1]),\n",
    "                user_age: np.reshape(x.take(3,1), [batch_size, 1]),\n",
    "                user_job: np.reshape(x.take(4,1), [batch_size, 1]),\n",
    "                movie_id: np.reshape(x.take(1,1), [batch_size, 1]),\n",
    "                movie_categories: categories,  #x.take(6,1)\n",
    "                movie_titles: titles,  #x.take(5,1)\n",
    "                targets: np.reshape(y, [batch_size, 1]),\n",
    "                dropout_keep_prob: dropout_keep, #dropout_keep\n",
    "                lr: learning_rate}\n",
    "\n",
    "            step, train_loss, summaries, _ = sess.run([global_step, loss, train_summary_op, train_op], feed)  #cost\n",
    "            losses['train'].append(train_loss)\n",
    "            train_summary_writer.add_summary(summaries, step)  #\n",
    "            \n",
    "            # Show every <show_every_n_batches> batches\n",
    "            if (epoch_i * (len(train_X) // batch_size) + batch_i) % show_every_n_batches == 0:\n",
    "                time_str = datetime.datetime.now().isoformat()\n",
    "                print('{}: Epoch {:>3} Batch {:>4}/{}   train_loss = {:.3f}'.format(\n",
    "                    time_str,\n",
    "                    epoch_i,\n",
    "                    batch_i,\n",
    "                    (len(train_X) // batch_size),\n",
    "                    train_loss))\n",
    "                \n",
    "        #使用测试数据的迭代\n",
    "        for batch_i  in range(len(test_X) // batch_size):\n",
    "            x, y = next(test_batches)\n",
    "            \n",
    "            categories = np.zeros([batch_size, 18])\n",
    "            for i in range(batch_size):\n",
    "                categories[i] = x.take(6,1)[i]\n",
    "\n",
    "            titles = np.zeros([batch_size, sentences_size])\n",
    "            for i in range(batch_size):\n",
    "                titles[i] = x.take(5,1)[i]\n",
    "\n",
    "            feed = {\n",
    "                uid: np.reshape(x.take(0,1), [batch_size, 1]),\n",
    "                user_gender: np.reshape(x.take(2,1), [batch_size, 1]),\n",
    "                user_age: np.reshape(x.take(3,1), [batch_size, 1]),\n",
    "                user_job: np.reshape(x.take(4,1), [batch_size, 1]),\n",
    "                movie_id: np.reshape(x.take(1,1), [batch_size, 1]),\n",
    "                movie_categories: categories,  #x.take(6,1)\n",
    "                movie_titles: titles,  #x.take(5,1)\n",
    "                targets: np.reshape(y, [batch_size, 1]),\n",
    "                dropout_keep_prob: 1,\n",
    "                lr: learning_rate}\n",
    "            \n",
    "            step, test_loss, summaries = sess.run([global_step, loss, inference_summary_op], feed)  #cost\n",
    "\n",
    "            #保存测试损失\n",
    "            losses['test'].append(test_loss)\n",
    "            inference_summary_writer.add_summary(summaries, step)  #\n",
    "\n",
    "            time_str = datetime.datetime.now().isoformat()\n",
    "            if (epoch_i * (len(test_X) // batch_size) + batch_i) % show_every_n_batches == 0:\n",
    "                print('{}: Epoch {:>3} Batch {:>4}/{}   test_loss = {:.3f}'.format(\n",
    "                    time_str,\n",
    "                    epoch_i,\n",
    "                    batch_i,\n",
    "                    (len(test_X) // batch_size),\n",
    "                    test_loss))\n",
    "\n",
    "    # Save Model\n",
    "    saver.save(sess, save_dir)  #, global_step=epoch_i\n",
    "    print('Model Trained and Saved')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存参数 save_dir 在生成预测时使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_params((save_dir))\n",
    "\n",
    "load_dir = load_params()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "显示训练Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvIAAAH0CAYAAABfKsnMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4FWX6xvH7TUICJKGE0Is06SDSwYYVxYYF28qqa/ntsrtWFlcFZRUFdVn72gVdWUUUwaWJCkqR3ntvIUCAQAjpyZnfHyc55CQ5JMiZM5nw/VwXV5KZOTNPgpH7vPPM+xrLsgQAAADAXcKcLgAAAADA6SPIAwAAAC5EkAcAAABciCAPAAAAuBBBHgAAAHAhgjwAAADgQgR5AAAAwIUI8gAAAIALEeQBAAAAFyLIAwAAAC5EkAcAAABciCAPAAAAuBBBHgAAAHAhgjwAAADgQgR5AAAAwIUI8gAAAIALRThdgB2MMTslVZO0y+FSAAAAULE1lXTcsqxmob5whQzykqpVqVIlrm3btnFOFwIAAICKa+PGjcrIyHDk2hU1yO9q27Zt3PLly52uAwAAABVY165dtWLFil1OXJseeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQR4AAABwoYo6jzwAACgDj8ej5ORkpaamKisrS5ZlOV0S4BhjjKKiohQbG6u4uDiFhZXvMW+CPAAAZymPx6O9e/cqPT3d6VKAcsGyLGVmZiozM1NpaWlq3LhxuQ7zBHkAAM5SycnJSk9PV0REhOrVq6fo6OhyHVoAu3k8HqWlpenAgQNKT09XcnKy4uPjnS4rIH5bAQA4S6WmpkqS6tWrp9jYWEI8znphYWGKjY1VvXr1JJ38HSmv+I0FAOAslZWVJUmKjo52uBKgfCn4nSj4HSmvCPIAAJylCh5sZSQe8GeMkaRy//A3v7kAAABAIQVBvrwjyAMAAAAuxKw1QWJZlvI8J2+/RITzHgkAAAD2CUraNMbcaox5yxgzzxhz3BhjGWM+L+U1fYwx040xycaYdGPMGmPMo8aY8GDUFGqZOR61fGaGWj4zQx1GfO90OQAAwEVOnDghY4yuu+66Mz5Xt27dFBMTE4Sqguftt9+WMUZff/2106VUKMEaNh4m6S+SOkvaV9rBxpgbJc2VdLGkbyW9IylS0muSvgxSTQAAAKdkjDmtP+PGjXO6ZMAnWK01j0lKkLRN0iWS5gQ60BhTTdKHkvIk9bUsa1n+9uGSZku61Rhzh2VZBHoAAGCr5557rti2119/XSkpKXrkkUdUo0YNv32dO3e2pY7o6Ght3LgxKCPp33zzTbmfNhHBEZQgb1mWL7iX4SnfWyXVlvRZQYjPP0emMWaYpJ8k/UkuHpkv5zMVAQCAfCNGjCi2bdy4cUpJSdGjjz6qpk2bhqQOY4zatGkTlHOdc845QTkPyj8nnsi8LP/jzBL2zZWULqmPMSYqdCWdOZfMUgQAAIKgoA89IyNDw4YNU8uWLRUZGam//OUvkqQjR45o9OjRuuSSS9SgQQNFRkaqbt26uuWWW7RixYpi5wvUIz9kyBAZY7Rs2TKNHz9eXbt2VZUqVRQfH69BgwYpKSkpYG2FTZ06VcYY/fOf/9SSJUvUr18/Va9eXTExMbriiiu0fPnyEr/PPXv26O6771Z8fLyqVq2qrl27asKECX7nO1MLFy7UjTfeqPj4eEVFRal58+Z69NFHdejQoWLHJiYm6pFHHlGrVq1UtWpV1axZU23bttX999+vvXv3+o7zeDz68MMP1bNnT8XHx6tKlSpq0qSJ+vfvr8mTJ59xzeWFE7PWtM7/uKXoDsuyco0xOyW1l9Rc0sZTncgYU/J/dVJw3tICAAAE4PF4dN1112nz5s3q16+fatWq5RsNX7lypZ577jn17dtXN954o6pXr66dO3fqu+++09SpU/XDDz/o4osvLvO1XnnlFU2dOlU33nijLr30Ui1YsECff/651q1bp2XLlik8vGxzhcyfP1/Dhg1T37599eCDD2rHjh2aPHmy+vbtq3Xr1vmN5ickJKh3795KTEzU5Zdfru7du2vfvn265557dM0115zeDyuAr776Sr/73e8UHh6ugQMHqlGjRlq0aJHeeOMNTZkyRQsWLFCDBg0kScePH1fPnj2VmJioq666SgMGDFBOTo52796tr7/+WoMGDVLjxo0lSY8++qjeeustnXvuubrzzjsVExOjxMRELV68WJMnT9aAAQOCUr/TnAjy1fM/pgTYX7C9RoD9AAAAjsvIyFBqaqrWrVtXrJe+S5cuOnDggGrWrOm3ffv27erZs6eeeOIJLV26tMzX+umnn7Rq1Sq1atVKknfa6wEDBui7777T999/r/79+5fpPFOmTNHEiRN16623+raNGTNGQ4YM0TvvvKNXXnnFt/2JJ55QYmKinn/+eQ0fPty3ffDgwbrwwgvLXHsgycnJeuCBB2SM0fz589WtWzffvuHDh2vkyJH6y1/+okmTJkmSpk2bpoSEBA0bNkwvvPCC37kyMzOVm5sr6eRofIsWLbR27VpFRfk3eRw+fPiMay8vyuM88gVNKqV2mluW1bXEE3hH6rsEs6jTQYs8AKAiaPr3aU6XUGa7Rl/ryHVHjRpVLMRLUlxcXInHt2jRQjfccIPGjh2rI0eOqFatWmW6zt/+9jdfiJe8PfUPPPCAvvvuOy1ZsqTMQb5fv35+IV6SHnroIQ0ZMkRLlizxbUtNTdWkSZNUp04d/e1vf/M7vlevXho4cKC+/PLMHmecOHGiUlNT9eCDD/qFeEl65pln9NFHH2nKlCk6fPiw4uPjffuqVKlS7FyVK1f2+9oYo8jIyBLvVBQ+l9s50SNfMOJePcD+akWOAwAAKJd69OgRcN+cOXN08803q1GjRoqMjPRNYTl27FhJ3n7vsioadCX52kiOHj16RueJjY1V9erV/c6zbt065ebmqmvXrsVCsqSgjMgXPCtw2WWXFdtXuXJl9enTRx6PR6tXr5YkXXnllapdu7aGDx+u6667Tu+8845WrVolj8fj99qwsDDdcccd2rhxozp06KDhw4dr1qxZSk1NPeOayxsnRuQ3S+omqZUkvx53Y0yEpGaSciXtCH1pAAAAZVO1alXFxsaWuO/zzz/X73//e8XExOjKK69Us2bNFB0dLWOMZs2apYULF57WFJEljfpHRHhjXF5e3hmdp+Bchc+TkuIdT61bt26JxwfafjoKrlG/fv0S9xdsP3bsmCTvSPrixYs1YsQITZ06VdOmTfPV8vDDD+vJJ5/0jcC///77atOmjT799FONHDlSklSpUiXdcMMNGjNmTIWZ2ceJID9b0u8kXS3piyL7LpZUVdJcy7LcOwEqvTUAgArAqXYVtzjVlNvDhg1TbGysVq5cqebNm/vt27p1qxYuXGh3eWekWjVvg8TBgwdL3B9o++moXt3bnHHgwIES9+/fv9/vOElq1qyZPv30U3k8Hq1bt04//fST3n77bT3zzDMKDw/Xk08+Kckb2ocOHaqhQ4fqwIEDmjdvnj7//HN988032rRpk1avXl3mB4TLMydaa76WdFjSHcYY3/0dY0xlSSPzv3zXgbrOCNNPAgAAScrNzdXu3bvVuXPnYiE+Jyen3Id4SerYsaMiIiK0fPlyZWZmFts/f/78M77G+eefL0n6+eefi+3LysrSwoULZYwpcRGusLAwderUSY899pimTp0qSQGnlaxXr54GDhyoKVOmqEePHlq/fr22bdt2xvWXB0EJ8saYAcaYccaYcZL+nr+5d8E2Y4xvklHLso5LelBSuKSfjTEfGWNekbRKUm95g/6EYNQFAAAQahEREWrYsKHWr1/vN0OKx+PRU089pZ07dzpYXdnExsZqwIABSkpK0quvvuq3b/HixZo4ceIZX+O2225TTEyMxo4d6+uDLzBq1Cjt37/fN7+8JK1atUoJCQnFzlNwd6Bq1aqSvHPy//LLL8WOy8rK8rXzlPTArBsFq7Wms6R7imxrnv9HknZLGlKww7KsycaYSyQ9I+kWSZUlbZP0uKQ3LYu1UQEAgHs99thjGjJkiDp16qSbb75ZYWFh+uWXX7Rr1y5dc801mjFjhtMllmrMmDGaP3++nn32Wc2dO1fdu3dXQkKCvvrqK11//fWaPHmywsJ++5hwXFycPvjgAw0aNEi9e/fWwIED1bBhQy1atEhz5sxR48aN9fbbb/uOnzp1qp577jldeOGFat26teLj47V7925NmTJF4eHhGjLEGzWPHTumvn37qkWLFurRo4eaNGmi9PR0zZw5U1u3btVdd92lJk2anPHPpzwISpC3LGuEpBGn+ZoFkso2V5LLWDTJAwBwVnv88ccVExOjt99+W5988omio6PVt29fffXVV/rwww9dEeSbNGmiRYsW6amnntL333+v+fPnq127dvr000+VkZGhyZMn+3rpf6s777xTTZo00ejRozV16lSlpqaqQYMG+utf/6phw4apTp06vmNvuOEGHTp0SPPmzdOkSZN04sQJ1a9fX9dff72eeOIJ34w8tWrV0ksvvaQ5c+Zo3rx5OnTokKpVq6Zzzz1XTz75pO65p+jYs3uZijj4bYxZ3qVLly6Blhu2Q3auR62GeX8pK4UbbX2xQr5HAQBUIBs3ehdQb9u2rcOVwG0eeeQRvfnmm5o/f74uuOACp8uxRVl/P7p27aoVK1asCLS+kZ2ceNgVAAAALlDSXPdLly7VBx98oAYNGqhnz54OVIUC5XFlVwAAAJQDbdu2VZcuXdS+fXtVrlxZmzdv9rUFvfPOO7657OEMfvo2qIDdSgAA4Cw0ePBgTZ8+XePHj9eJEydUs2ZNXXfddRo6dKj69OnjdHlnPYJ8kDCPPAAAqGhGjRqlUaNGOV0GAqBHHgAAAHAhgrwN6KwBAACA3QjyQUJnDQAAQMXglunZCfIAAJylTP4DXh6Px+FKgPKlIMibcv4QJEEeAICzVFRUlCQpLS3N4UqA8qXgd6Lgd6S8IsjbwC23YwAAZ7fY2FhJ0oEDB5SamiqPx8O/YThrWZYlj8ej1NRUHThwQNLJ35Hyiukng6S833oBAKCouLg4paWlKT09XQkJCU6XA5QrVatWVVxcnNNlnBJBHgCAs1RYWJgaN26s5ORkpaamKisrixF5nNWMMYqKilJsbKzi4uIUFla+m1cI8jbgf4EAALcICwtTfHy84uPjnS4FwGkq328zXITGGgAAAIQSQR4AAABwIYI8AAAA4EIEeRvwnBAAAADsRpAPEmafBAAAQCgR5AEAAAAXIsgDAAAALkSQBwAAAFyIIB8khiZ5AAAAhBBBHgAAAHAhgrxNLOagBAAAgI0I8gAAAIALEeQBAAAAFyLIAwAAAC5EkLcJLfIAAACwE0E+iJiBEgAAAKFCkAcAAABciCAPAAAAuBBB3ia0yAMAAMBOBPkgokUeAAAAoUKQBwAAAFyIIG8Ti/knAQAAYCOCfBAZ5p8EAABAiBDkAQAAABciyAMAAAAuRJC3CR3yAAAAsBNBPojokAcAAECoEOQBAAAAFyLI24TZJwEAAGAngnwQMfskAAAAQoUgDwAAALgQQR4AAABwIYK8TSwmoAQAAICNCPJBZJiAEgAAACFCkAcAAABciCAPAAAAuBBB3ibMIw8AAAA7EeSDiRZ5AAAAhAhBHgAAAHAhgjwAAADgQgT5IKKzBgAAAKFCkAcAAABciCAPAAAAuBBB3iZMPwkAAAA7EeSDyNAkDwAAgBAhyAMAAAAuRJC3iSV6awAAAGAfgnwQGSagBAAAQIgQ5AEAAAAXIsgDAAAALkSQtwnTTwIAAMBOBPkgYvpJAAAAhApBHgAAAHAhgjwAAADgQo4GeWPMtcaYWcaYBGNMhjFmhzFmojGmt5N1BQMt8gAAALCTY0HeGPOypKmSukiaKekNSSsk3ShpgTHmbqdq+61okQcAAECoRDhxUWNMPUlDJB2U1MmyrKRC+y6VNFvS85I+d6I+AAAAoLxzakT+nPxrLy4c4iXJsqw5klIl1XaisGCxmH8SAAAANnIqyG+VlC2phzEmvvAOY8zFkmIl/ehEYWfCMP8kAAAAQsSR1hrLspKNMU9K+pekDcaYyZKOSGoh6QZJP0j6PydqAwAAANzAkSAvSZZlvW6M2SXpE0kPFtq1TdK4oi03JTHGLA+wq82ZVwgAAACUX07OWjNU0teSxsk7Eh8tqaukHZLGG2Necaq2YKBDHgAAAHZyataavpJelvStZVmPF9q1whhzk6Qtkp4wxrxnWdaOQOexLKtrgPMvl3day5CiQx4AAACh4tSI/HX5H+cU3WFZVrqkJfLWdn4oiwIAAADcwqkgH5X/MdAUkwXbs0NQCwAAAOA6TgX5efkfHzLGNCy8wxhzjaQLJGVK+jXUhQUL08gDAADATk7NWvO1vPPEXyFpozHmW0kHJLWVt+3GSPq7ZVlHHKrvt6FJHgAAACHi1DzyHmNMf0l/lnSHpJskVZWULGm6pDcty5rlRG0AAACAGzg5j3yOpNfz/1Q8tNYAAADARo7NI18R0VkDAACAUCHIAwAAAC5EkAcAAABciCBvE4smeQAAANiIIB9ExtAlDwAAgNAgyAMAAAAuRJC3CSu7AgAAwE4E+SCiswYAAAChQpAHAAAAXIggDwAAALgQQd4mtMgDAADATgT5IKJFHgAAAKFCkAcAAABciCAPAAAAuBBB3iYWE8kDAADARgT5IDJMJA8AAIAQIcgDAAAALkSQtwmNNQAAALATQT6IaKwBAABAqBDkg+hIWrbTJQAAAOAsQZC3yZ7kdKdLAAAAQAVGkLfJtDX7nS4BAAAAFRhB3iZMIw8AAAA7EeRtYjFvDQAAAGxEkAcAAABciCBvE1prAAAAYCeCPAAAAOBCBHmbWAzJAwAAwEYEeQAAAMCFCPI2YTweAAAAdiLI24TOGgAAANiJIA8AAAC4EEEeAAAAcCGCvE089NYAAADARgR5mxDjAQAAYCeCPAAAAOBCBHmb0FkDAAAAOxHkbUOSBwAAgH0I8gAAAIALEeRtQmsNAAAA7ESQtwlBHgAAAHYiyNvEokceAAAANiLIAwAAAC5EkAcAAABciCBvEw+dNQAAALARQd4mPOwKAAAAOxHkAQAAABciyNuEWWsAAABgJ4K8XcjxAAAAsBFBHgAAAHAhgjwAAADgQgR5m9BZAwAAADsR5G1iMf8kAAAAbESQBwAAAFyIIG8TxuMBAABgJ4K8TeisAQAAgJ0I8gAAAIALEeQBAAAAFyLI24TOGgAAANiJIA8AAAC4EEEeAAAAcCGCvE1YEAoAAAB2IsjbhBgPAAAAOxHkAQAAABciyNuFIXkAAADYiCAPAAAAuBBB3iYWQ/IAAACwEUEeAAAAcCHHg7wx5iJjzDfGmP3GmKz8j7OMMf2drg0AAAAoryKcvLgxZpikFyQdljRV0n5J8ZLOl9RX0nTHijtDTCMPAAAAOzkW5I0xA+UN8T9KutmyrNQi+ys5UhgAAADgAo601hhjwiS9LCld0l1FQ7wkWZaVE/LCgogReQAAANjJqRH5PpKaSfpa0lFjzLWSOkjKlLTEsqyFDtUFAAAAuIJTQb57/seDklZI6lh4pzFmrqRbLcs6FOrCgoXpJwEAAGAnp4J8nfyPf5S0U9IVkhZLOkfSGEn9JE2U94HXgIwxywPsahOUKgEAAIByyqnpJ8PzPxp5R95/sizrhGVZ6yXdJClB0iXGmN4O1XfG6JEHAACAnZwakT+a/3GHZVmrC++wLCvDGPO9pPsl9ZAUsF/esqyuJW3PH6nvEqRaAQAAgHLHqRH5zfkfjwXYXxD0q4SgFgAAAMB1nArycyXlSjrXGBNZwv4O+R93hayiIKOzBgAAAHZyJMhblnVY0gRJ1SU9W3ifMeZKeR92TZE0M/TVBQc98gAAALCTYyu7SnpcUk9JzxhjLpa0RN5Za26SlCfpQcuyArXeAAAAAGc1x4K8ZVlJxpiekobJG957SUqVNE3SKMuyFjlVGwAAAFDeOTkiL8uykuUdmX/cyTrsEBFmnC4BAAAAFZhTD7tWeBe1ine6BAAAAFRgBPkgurlLQ9/n4YYReQAAANiHIB9ElcL4cQIAACA0SJ42YfZJAAAA2IkgH0SFu2mYRx4AAAB2IsgHEW3xAAAACBWCvE0smmsAAABgI4J8UDEkDwAAgNAgyNuEHnkAAADYiSAfRPTIAwAAIFQI8jZhQB4AAAB2IsgHkd+APL01AAAAsBFBPohorQEAAECoEORtwng8AAAA7ESQDyLD9JMAAAAIEYK8TWiRBwAAgJ0I8kFEjzwAAABChSBvE4sheQAAANiIIB9EDMgDAAAgVAjyNmE8HgAAAHYiyAeRKdQkT2cNAAAA7ESQBwAAAFyIIG8TBuQBAABgJ4J8EDH9JAAAAEKFIG8Tpp8EAACAnQjyQWSYgBIAAAAhQpAHAAAAXIggH0SFe+TprAEAAICdCPJBRGMNAAAAQoUgbxOLCSgBAABgI4J8EC3bfdT3+aIdyQ5WAgAAgIqOIB9Eq/Ye830+e1OSg5UAAACgoiPIAwAAAC5EkAcAAABciCAPAAAAuBBBHgAAAHAhgjwAAADgQgT5IOrTopbv88va1HGwEgAAAFR0BPkg6tX8ZJBv36Cag5UAAACgoiPIAwAAAC5EkAcAAABciCBvE8tyugIAAABUZAR5mxzLyHa6BAAAAFRgBPkg+n79Ad/nny/a42AlAAAAqOgI8kG0PvG40yUAAADgLEGQBwAAAFyIIA8AAAC4EEEeAAAAcCGCPAAAAOBCBHkAAADAhQjyAAAAgAsR5AEAAAAXIsgDAAAALkSQBwAAAFyIIA8AAAC4EEEeAAAAcCGCPAAAAOBCBHkAAADAhQjyAAAAgAsR5AEAAAAXIsgDAAAALkSQBwAAAFyIIA8AAAC4EEEeAAAAcCGCPAAAAOBCBHkAAADAhQjyAAAAgAsR5AEAAAAXKjdB3hgzyBhj5f95wOl6AAAAgPKsXAR5Y0xjSW9JOuF0LQAAAIAbOB7kjTFG0lhJRyS953A5AAAAgCs4HuQlPSzpMkn3SUpzuBYAAADAFRwN8saYtpJGS3rDsqy5TtYCAAAAuIljQd4YEyHpP5L2SHraqToAAAAAN4pw8NrPSjpf0oWWZWX8lhMYY5YH2NXmN1cFAAAAuIAjI/LGmB7yjsKPsSxroRM1AAAAAG4W8hH5Qi01WyQNP5NzWZbVNcA1lkvqcibnBgAAAMozJ0bkYyS1ktRWUmahRaAsSc/lH/Nh/rbXHagPAAAAKPec6JHPkvRxgH1d5O2bny9psyRXt92kZuYotnIlp8sAAABABRTyIJ//YOsDJe0zxoyQN8h/alnWR6Gsyw7p2XkEeQAAANiiPCwIVWFZltMVAAAAoKIiyAMAAAAuVK6CvGVZIyzLMhWhrUaSLDEkDwAAAHuUqyDvdj2axfl9TWsNAAAA7EKQD6I/XNDM6RIAAABwliDIB1HlSv4/TgbkAQAAYBeCvI0semsAAABgE4J8EBljnC4BAAAAZwmCfBAVjfEMyAMAAMAuBHkAAADAhQjyQURnDQAAAEKFIB9EpkhzDa01AAAAsAtB3kas7AoAAAC7EOQBAAAAFyLIB1HRHnlaawAAAGAXgnwQFX3WdcnOZEfqAAAAQMVHkA+mIkl+y8FUZ+oAAABAhUeQBwAAAFyIIB9EEWH8OAEAABAaJM8g6tCwmt/XrevFOlQJAAAAKjqCfBCFFZm2Jjkt26FKAAAAUNER5G00asYmp0sAAABABUWQD6Ki88gDAAAAdiHIB1HR1hoAAADALgT5ICLGAwAAIFQI8kHEiDwAAABChSAfROR4AAAAhApBPogMSR4AAAAhQpAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggDwAAALgQQR4AAABwIYI8AAAA4EIEeQAAAMCFCPJBVrkSP1IAAADYj9QZZEbG6RIAAABwFiDIB1meZTldAgAAAM4CBPkgy871OF0CAAAAzgIEeQAAAMCFCPIAAACACxHkAQAAABciyAMAAAAuRJAHAAAAXIggbzOL6SgBAABgA4K8zTzkeAAAANiAIG8zRuQBAABgB4K8zVjpFQAAAHYgyNts1Z5jTpcAAACACoggb7PMXI/TJQAAAKACIsjbLD0r1+kSAAAAUAER5G32p/ErdDwzx+kyAAAAUMEQ5EPgndnbnC4BAAAAFQxBPsha1I4utu3QiSwHKgEAAEBFRpAPsueub+90CQAAADgLEOSDrFZMZPGNTCUPAACAICPIh8CWpFSnSwAAAEAFQ5APgXX7jjtdAgAAACoYgnyQ1a1W2ekSAAAAcBYgyAdZfExUiduzWeEVAAAAQUSQD5GP5+90ugQAAABUIAT5EJm2NtHpEgAAAFCBEOQBAAAAFyLIh4jFXPIAAAAIIoI8AAAA4EKOBHljTC1jzAPGmG+NMduMMRnGmBRjzHxjzP3GmAr3BiPxWIbTJQAAAKACiXDougMlvStpv6Q5kvZIqivpZkkfSbrGGDPQsipOQ8rR9BynSwAAAEAF4lSQ3yLpBknTLMvyTbBujHla0hJJt8gb6r9xpjwAAACgfHOkhcWyrNmWZf2vcIjP335A0nv5X/YNeWEAAACAS5THXvSCHpRcR6sAAAAAyrFyFeSNMRGSfp//5Uwna7HD/1YnKifPU/qBAAAAQCmc6pEPZLSkDpKmW5b1fWkHG2OWB9jVJqhVBclfv1ip529sr9/3bup0KQAAAHC5cjMib4x5WNITkjZJGuRwOWfkxZs6BNz37JT1IawEAAAAFVW5GJE3xvxZ0huSNki63LKs5LK8zrKsrgHOt1xSl+BVeHoualnbqUsDAADgLOH4iLwx5lFJb0taJ+nS/JlrXM2YU+8/kJKp/yzcpX0sEgUAAIDfyNEgb4x5UtJrklbJG+KTnKwnWMLCTp3kH/rPMg2fsl73j1uqCrTmFQAAAELIsSBvjBku78Oty+VtpznsVC3BVkqO15qEFEnSpgOpyvUQ5AEAAHD6HOn2yao4AAAgAElEQVSRN8bcI+l5SXmS5kl62BTvR9llWda4EJcWFEalJPlCGJAHAADAb+HUw67N8j+GS3o0wDG/SBoXkmqCrLQR+cIycvIUGeH4owoAAABwGUcSpGVZIyzLMqX86etEbUFxGkH+nTnb7KsDAAAAFRZDwQ6buGyv0yUAAADAhQjyNjidHvmj6TnKyM7T3uR0GysCAABARUOQLwfaPjtTF70yR18tDTw6vz8lg6kqAQAA4EOQL0eGfrOmxO2jZmxU71Gzde/YpSGuCAAAAOUVQd4Gpa3seio/b07S5WN+1sipG3zb3v9lhyTply2HtOtw2pmWBwAAgArAqeknK7QzyPG+Uffth3bqhs4N1KlRDb/96dl5Z3B2AAAAVBSMyNugcqXwoJzn1+1Him3zWBa98gAAACDI2yE6KkJRQVjkafSMTcW2vTxzk85/4Qd9tnDXGZ8fAAAA7kWQt8naEf3UoWG1Mz7Pje8s8Pt63tbDOpaeo2enrNfB45m65d1fddv7C3XkRJZSMnLOaLR+z5F0vTR9o+ZuOXSmZQMAAMBmpiK2aRhjlnfp0qXL8uXLHa3Dsiw1e2p6yK4XHmbUo2mc/vtgT5kAT9ympOfo0IlMtawTW2zf1a/P1aYDqZKk5cOuUK2YqFKvmZmTpxHfrVdadp5GXN+uTK8pkJqZo88W7la9apV1S9dGZX4dAABAedG1a1etWLFihWVZXUN9bR52tVGgMG2XPI+lhTuO6LUftujxq1oX2380LVsXvjxbadl5+ufA83RrkfBcEOIl6cXpG/Wv2zoHvFZSaqae/98GTV2z32/7W3eeX+Z6x8zaonG/7pIk1a9RWX1axJf5tW6RnetRZBDarAAAAIoiYVRAb87eprs+XKSM7DxtPZiqa9+cp79/s0ZjftistPxZb4ZMXH3Kc0xasU/jFuxUTp6nxP3Dvl1XLMT/b3XiKc/50vSN6vnSj5q8cp8k+UK8JN314WLtOHSitG/NNTweS3d+sEidn5+laUV+TgAAAMFAkK+gft1+RM9PXa8rX5ur9YnH9eXSvfp80R6/Y45n5pzyHCP+t0FPT1qrmesO6Jcth5Sb59G3KxM0eeU+zdpwsMTX/BBg++4jafpg7g4dPJ6lRyesKvGYy8b8og/n7vDblpqZo583Jykzx/sGJDMnT+MX7w54nfJi6tr9WrjjiNKz8/Tn/65wuhwAAFAB0VpTgX2xZO8p93caMUuSNPbe7ooP0Ns+cXmCJi5PkCS1b1BN6xOPn/KcD362TLtGX1ts+8HjWX5fFwTzol6cvlEPXtxckvcZg9vfX6QN+4+rX/u6en9QN73/yw699uMWSdKkwX3UpUlNSd4RcGNO3c6UeCxDURFhvj7+pOOZ+mlTks6tE6N9xzJ0Zbu6qhp5er8SlmVpfeJxtawTo4zsPFWKCFNMVIQSjqaf1nkAAABOF0Eeum/c0jIdV1qIL+xEVq4e+myZft1+RE9d00bn1o3x2//x/J0BX1sQyhOOZmjDfu81v1/vHYEvCPGS9PKMTZrwf72150i6Bn2yWOHG6L8P9lK96pWLnfPX7Yd190eLFREWppmPXqRm8dF64LNlWpOQ4jvm9m6N9fKtncr8PUrSi9M26qP87yUyPEyREWGa8chFMkWWBbMsK+TPTAAAgIqN1hoE3f6UDN314SLfglajZmzSH8Yt8ztm7IJdAV9/yT/n6LIxv2juVv9pMA+fKD6qf9/YJbr41TnafSRdOw6n6e+T1ijPY2nOpiSt23cypA/6eIk8lpSd59ETE1fLsuQX4iVpwrLidzA27j+uq1+fqwc+Xabs3OLPC3xU6A1Jdp5HJ7JyNfTrNSqa2X/amBTw+y3w2g9bdPXrc/XRvB22L/qV56l4s1UBAHC2YUQeQdd71OxSjykaygvbm5whSXrm23V+21fvPeb/dZEgLkmLdyRrwtK9evrbtZKkHx+/RC3rxPgF14MpmQoUY9/9ebsmr9ynTo2q67ErW+n+cUuVmJKpTQdSNXbBTv3fJS1K/d4W7jiii1vV9tt2JC3w9ytJ25JS9cZPWyVJI6dt1N7kdP3jxg6lXuu3GDx+uaavPaDzGtfQlD9fUObXncldhaNp2apRtRJ3JQAACCJG5G026uaOTpdQYdz/6bJSj8nIyfOFeEm64l+/6ERWrt8xiSmZSkrNLPH1L8/cpM0HUzVxeYL6jJ6txJSTx42asUkHCn19qlHtl2f6r8pb2gD4jkNpfl9/unC31icWf6Oy5WCqnvl2rb5auvc3jarvO5ah6WsPSPK+MVq552ipr8nKzdOoGRvV5YUf9Pmi3aUeb1mWsnJPPgPx9uyt6jLyhzL9/QEAgLIjyNvszh5NtHnk1YqPiXS6lLPWI1+sLLatLHcNStJr1E9auitZz/9vg1o8XfbFvp6atFZJxzOVcDRdU9ckan1iij79dZeSjnvfGISHFR+pvvbN+Rq/+GRwzsnz6KrX5mr84j0a+s0afThvh/anZOiBT5dp6NertedIuj6ev1ObDhzXDxsO6lBq8bsAR9Oy/b7+Kr+daOWeo7rzg0V6K/+uQIEJS/eo9bCZev+XHTqanqNhk9fpbxNXy+OxlJvn0b9+2KLn/7dBKRneGZBOZOXqytfmqudL3p+TJP1z1hZZljR7U5Jemr5RXyzZE3BaU7dJz87VsMlr9dSkNaXOAgUAQLDRWhMCURHh+uZPfXTJqz87XcpZ6adNpfenn46B7y38Ta/r8dJPxbY999167RzVX2v3FR99l7ztRTWrRmra2v2qXWRmodEzNmne1kNasM37LMJXyxKKvX7moxfp5RmblJKRo9dvP1+eIr33XyzZq4cubqGb/v2rJG9bUN/WddSxUXVJ0pPfrC12zonLE9S9aZyy8jx6Mz/453o8ev7GDnr9hy3aluRdD+D29xdqxyj/GYw+yJ9edOvBEzqSlqXuTeN0d69zSvze3eDdn7f7pnWNigjXiBval+l16dm5Op6RW+KD2QAAlBUj8iFyTq1oNYmr6nQZKIdmb0o65WJag8ev0LQ1+/0W0CpQEOIDufr1eZqz+ZBW7DmmRyasLNbCI0mX/vNnv69XJRwr9WHb+dsO64O5231ff7bQe+dg88GTqwN7LCk3wMj7Jwt2asqqRA2bvE4b95dtNiTLsrR0V3KxuwqFJadl66N5O0ptGbIsSwu2HdYPGw6WuUWppOPemr3N93lJfz8lOZqWrd6jZqvP6J/0/foDZXoNAAAlIciHUGQEP24Ud/+ny7S9hIAdbCv3HAu4GFdhwyevU7Onpuu5KesCHuOxLN9DyQV+99EipWb6P48wswxBdcG2w0o4mq5dh0/+DA6fyNLSXcl+byje+GmrBr63UJeO+VkZ2Xmasmqfer70o16YusF3zLDJazVy2kbd9O9fdSz9ZODPzMlTt5E/qunfp2nulkP6bnWifvfRYj342TJNXZOo9OxcTV+7P+CzE898u1adRnyv/5ThGYHSDB6/QikZOfJY0v/9Z3mJbUZH07L1+Fer9PS3a5WRXfKaC07Kys3TN8sT9PPm4N7tAgCcHpJlCHVqWN3pEoAy+3Rh4NA6dc3+YtsWbDuiVUVmFtp9pPSFsaasStSFL89R33/+rF+3HVZqZo66jfxRA99bqCe+Wq2cPI8em7BKr//obeM5lp6jL5fu0SNfrtLB41n6eP5O7TjkbecpeJBXkmau837+8fydajN8pm+mpN9/skSPfHnyDc0jX67S3yau0eDxK3Truwt1PDNHr/+4RWMX7JTHY2n3kTSNX7xHadl5Gj55nXYfSQvY45+d6znl3YxtSSe0cIf/XZQLX57t9xC15J25aNKKffrv4j1675ftKsmx9Gwt2HY44F0Py7KU+hv69rNzPVqw7bDSijwkXth/F+/RExNX696xS31/55/M36mb/r1AG05jvQkAwJkxds9X7QRjzPIuXbp0Wb58udOl+Hn8q1WatGKf7+uLzo3XvK2HHawIKH+GXNVK/5y15ZTHdGhYTev2nTowjrq5o+ZsStKsDQdP6/q9m9fyhe037zxfjWtW8T1DUFid2CglFXmgODYqQi3rxuizP/RQWlaeqkaFK+l4llrW8S6Idvv7C7V4Z3Kxc13QspY++n13Zed5VL1KJTX9+zS//ev/0U/RURE6eDxTXy7Zq67n1NTg8ct1PDNXfVvX1rj7evgdn5Gdp7bPzpQktaobo1mPXSJJWrX3mKatSdQtXRupTb1qxeqYvHKf765Nu/rVNO3hC0ucMrRwfZ0b19CTV7fRnR8u8m3r2SxObetXU89mcZqwbK/u6d1Ul7apU+w8v9WCbYf13apE3dmziTo3rhG0825LStX+lExd0CJeYSU8gH62y8nz6KeNSWoWH61z68Ro88FUtaobW+LD+sDZpGvXrlqxYsUKy7K6hvraBPkQWror2fegZL/2dfX+oG7F/sEGznY9msVpSQlh1wlVI8P1+QM9dXMJQf50vHJrJ93WrXGZft8//UMP3fPJEr9tlSuF6W/92vi1ERW28fmrVSUy3Pf1v2Zt1puF+veXDbtCNapUUstnZkjyvuFY+49+fudYvvuobnnX//v8eUhf1a9RWbPWH1STuKo6Lz80F/4+zmtUvcQ1HYraNfraUo8pi6zcPLUeNjPgeS3LUk6e5WtlLOv6B3uT03Xxq3NkWdILAzpoUKGHsDOy87TzcJra1o8t81oIx9Kz9cZPWxVXNVKDL21ZYtjNzvVozuYkdWhYXQ1rVNGx9GzVqBqaGc5emblJszcl6an+bXVJkXUvCuTkee/OnNeohmpGR+rfP2/TKzM3KzzMqHvTmlq0I1kXt6qtz/7Qo8TXw53Ss3M1b+th9WpWS9WrVnK6HFdwMsgza00IdW8ap9E3d9T2Qyf00MXehYX++0BP3fXRYocrA8qP8hLiJSk9O++MQ7wkDf16jW7r1rhMxxYN8ZKUmeMJGOIlaeb6/erRrJZqVKmk6KgIvxAvSd1G/qg7up+8fmpWruZtPaTuTeNUuVK4Xv9xi691qbAlO5M19Js1kiRjpLfv7KKLW8X7HXMmQ0FH8tudahWakenXbYf19fIE3da9sXo1ryXJ+9B0RHiYMrLzAi6uZlmWjqbnqMsLP0iSJjzUS1uSTmj45HW6pkM9vXt3V42cukEz1h3QM9e2Vf+O9f1e/+K0jSoY1xo+eZ0vyOfkeXx3NyRpzYirNHfLIV3QIl41owOH7tEzNunLpd7pXX/Zckgjb+pQ7C7I6Bmb9MkC7+rQ3c6pqWW7j+qm8xvqtds7l3jO9OxcVY0883+2NyQe179/9rZs3fPJEt+boX3HMtSgemXfm5Xhk9fpy6V71bBGFf3yt756ZeZmSd4Hvxft8P6ezt1ySCkZOapexR2Bb29yuoZ+vUZx0ZEac9t5qlwpvPQXVXDZuR5tTUpVu/rVZIzRw1+s1I8bk9SmXqxevfU8rdp7VDd0buiav+OzDUE+xO7o0cTv6z4t49WreZzvf4oAKqYeL/5o27kfm7Da9/mfLy159eGCUFlg0MdLFGakpc9cUWKIl+QL8ZJkWdKf/7ui2DFryjAaL3lHtQvfNVi3L0UD3lmgXI+lhy8/V98sT1DrerGanT9d7KSV+7Rr9LUa+vVqv6lViw6IZ+d61GrYjGLXu/2Dk60+M9Yd0AtTN+jj+d7QPHj8Cn39x96auCxBN3ZuoD4t4/0WMTv5PVu+KVMLdBoxy/f54qcvV2zlCO08nKaqkRFqFh8tyfuwcuGf97LdR3XLv3/V4meuUEzUyX92C0J8wTGS9O3KfbqhcwMt2nFEV7atq86NaygiPEyfLdylF6Zu0EXn1tbH93Tzhe2XZ3qnoX26f1v1aeH/JiuQX7cXb+kcOXWDPpq/U5e2rq2x9/VQZk6e73vYdyyj2LMdhe1PydCGxOPq2SwuaC1JlmXpxWkbtSXphJ69rq1a1okNeGxOnkcJRzN8P/9TeeTLlVqxx/tcx7l1Y/ToFa3OqE6Px9LKvcfUpl6sqkaGl3jHZufhNP208aCu6VhfEWFG6/al6KJza9s+AYbHY2n8kj06npGje/s0VXRU8chnWZZufe9XrUlI0aBe5+iFAR3040bv7+CmA6m6/u35kqRVe1M05rbzbK0Xvw2tNeXAweOZ+mZFgm+0AwAqorjoSF3SqrYeuKiZ7vpwsW8hsTNxe7fGmrBsb+kHnsJHv++mBz4785WH/3hJC+07lqH1+1K043DxmaiGXNVKEeFhio+J0i1dGqrZU2VfVK6wb/7UW13PidPKPUf9nt/YPPJqRUWEa8vBVG1POqHL29YtFhZ/3Xa42F3gXaOvLbXta+x93XXf2KWnPGbo1a01uG9Lv23LdyfrlncXKjI8THOHXqqk1Ex1bFhdxhjleSy/lqOjadnaeOC4alaN1NakE3o4fzG/xnFVNG/oZb7j9qdk6NZ3F6pPi1oaeVMH9X9jnrYfStNlbero0SvOVadGgZ+bKPx9tm9QTdMevkjbkk5ofWKKasdEaf62wzp8Iku1YqKUnevRfRc0VaOagaeOfv5/G3xvyGpFR+rRK1tpUK9zfG9cPR5LfUbP1oHjmWpeO1pH07J1ND1H9/Q+R/uOZWhb0gmNue08dT0nTpL0rx+26JvlCXriqla6uUsj33XWJqRo5vr9uun8Rr5nbkozfe1+DR7vffM9uG8LDb26jb5Yskc/b07SXy49Vx0bVdeGxOPq/+Y832tO9d9CsNrjCsvKzdPEZQmqUilcN53f8JRvBHPzPMr1WOXyLgqtNWe5utUqa3DfljqYknnKmUIAwM2S07L17cp9+nblvtIPLqMzDfGSghLiJQWcYahA4Ye4q0b+9jBy+IR3atUR//Nvt2o9bKaWPHO5rnptriSpU6Pq6t+xvu7o3ljVKlfSXR8tKvHu712FHlQOJDu39NWYX5m5Wf3a11OL2ieD5i3vep8Ly87zqNco76J4z13fTrM3JWntvhS9cGMHXd2hnqauSfS7s1RYwVS3ny3cpUU7jvhmp5q4PEGJKRm+6Xtnb0rS7E1JalknRvf2aapm8dH6fNFu3dylka5sV7fEc6dk5Oj6t+YrI6fkaV6X7UrW5D9fUOJIe2ZOnt9dlSNp2Ro+eZ08Hksjp23Qpa3raNTNHXUgfwXvwut4FP63/pZ3F+q2bo3018vO9S2y9/hXq31BPifP4xsZn7B0r5YNu9J7vRNZCg8zAZ+reP3Hk/+9/fvn7bqjexM9Ncm7yN+czYc0ckAHDf16TYmvLUlaVq5+2XJIDWtU8T0vUxZJqZmavHKfLmgZr+pVKmnsgl2KCDcyMsrMyfOtwVE1MlzXFGl5k7zf/7H0HPV/c54OpWbpvMY19OBFzXRdpwZlrqEiY0S+nDmalq1r35ynxPzp6F67/Txd2LK29h3L0IB3FjhcHQCgPHjiylYa88OpZ3cqEB0ZruduaH9aoe1MbHi+n6+Xv7xM6HDRufG6vXtj/eW/K3/T63/Xs4n+fGlL1YmN0rGMHB3PyNGYH7ZoWglT8RY29t7uum/cqe9kBLL46ctVt1plPfDpUl+7iyQ9cGEzRUdF6I380H9JK2+7VUT4ybsvWbl5umD0HN+0u2W1+tmrdN7zs0o97sfHL/Zrd3ros2WateGgYqIi9Nz17VS5Uriu7VhfYWFGd3+0WPO3HVZkRJhy8jwKFDvjY6K0bNgVkrxrifyw4aDmbjmkGetKXo9kwkO91LN5LZ3IytXI/GeIHr2ilSMrZjNrTZC5OchL0u4jaXr35+3qck5Nvwfkysv/EAEAOJWr29fTkH6tdMW/5jpdStCFh5kyrwh9poZf1+6UD7oXiAgz+vWpy1QntrLembNNr37/21p1b+7S0G+a7EAKz1ZkWVaJbWKv3tpJ368/qB83lm0K4PiYSH3/6MX60/gVZZ70YFCvc/wW6rugZS2Nf6BXmV4bTAT5IHN7kA/kVEH+973PUfemccrzWOrZPE7Ldh3V7E1JQb2FLUkPX37y1h8AADj71KhaSff2aarqVSrpirZ1ddErc874nHHRkUpOyy79wFLY0ctfGoJ8kFXUID9q+ka9X2QGhQLfDu6j85vULHHf5gOpysjJU5VK4Rq7YKffbArXdapf4iqdgewafW2xeZwBAADKg7MtyPOwq4v8/Zo2uqFzAx05ka1ZGw7o0tZ1NG/rYbWtHxswxEtS63on+9ieuqatqkSGq3qVSvrjJS1UuVK43rzDUvOnyz57QlREuLo0qeGbwqsk2168xrf4TIGNz1+tAe8s0OaDqWW+FgAAQFnl5HlUKdzeqT3Lk7PnO60AjDFq36C6Lm5VWyMHdNTlbetqxA3tdXv3JqW/OF/1qpX03PXt9egVrXxTOIWFGT2WP5dulyY1NPGPvXVdp/p69rp2fq8dctXJ+XbfuON8Xdq6turERqmk2aIiwsP00k0dfV+/eFMHVYkMV+8WtYod26NZnGoUWT0u0FzYknRvn6b6wwXNdGePxnrrzvP1698vC3gsAAA4exSsF3G2oLUGPonHMlSvWmW/eVyzcvN0KDVLmTmegHPXHjmRpSqR4Xrvlx2avHKfhvRrrRvO804LlXA0XbVjoxQV4X3TkJqZo0EfL9H6xBR1bFhdN3VppEG9zinWrrP1xWs0arp31cPGcVV8049J0qibO+rOIgtrfTB3u16avum0v+eXb+mohjWq6u6PWV0XAICKINTtNbTWoFxoUKNKsW1REeGnXAxDOrm8+uNXttLjV/qvklf0tbGVK2nyny+QZVl+8/JGhPnfHKoUHqZnr2+nZ65tKyP5tf6UNP/yoF5N9caPW5WWnaf+HeupepVIfbFkT4n1tqtfTc9e306t68aqZnSkLMvSuPu6695SFjs5lRcGdFDXJjX1+eLd+u/ik9eNjYrQ2n/007akVI1fvEdjF+z6zdcAAAAojNYaOKLo4hrhYUb/uu08dW5cQ2/c0dlve1iY0V8v864W2KB6ZfUvYcGIKpHhmjT4Ar1wY3u9OKCj+nesF/DaX/+pt3o1r6Wa0ZG+Wvq2rqMFf79McdHFF9YYOaCDxt3XXRFFeojioiPVvHa0Nj5/tQb1OkftGlTzaycqqEuSWtaJ1XPXt9fOUf01sGsjNQ+wlHjRa3zzp94Bv49A2tWvdtqvAQAA7sOIPMqNm7s08luSurDHr2ylK9vVVYvaMQEfYmldL9b3YO9F59bW0Ktba+vBE35TcJ5Tq6pvoZKiGtaoooVPXaZVe47p1e83q0PD6nru+na+Nx2bR14jSdp04Lha1fVeJyLMlLjiX4GwIvuMMXp14Hm+r5/+dq3fCP4rt3bS41+dXN2wYNnuAnf2aOK707D6uau0aMcR/brtsC5tU0eREWHq3byWjDEa8d16jft1l+pXr6yJf+ytCUv3KswY3wIipXnl1k56atLaUudK3vFS/9N6UBoAAAQPQR6uYIxRp0ZlXxJakgb39Y7iG0mT8sP873qe+sHgqIhw9WxeS1//qU+xfeH5o+XtG1Qvcw2nyPiSpBcHdNCK3Ue16UCqOjeuUeJzCE/3b6O3ftqme/o01ZB+rfXCje19K/j1a19P/doXv/sw/Lp2uqpdXbVvUF3Vq1bSE1e1Vk6eR+MX79bhE9lqFh+tnYfT/F7z8GUtNbBbYzWO87ZD9W1dW9UqV9I//rchYJtSWJE7CFtfvEZpWbnq/PwPxY6d8FAvLdt91G+hknrVKvuWLw8kzEjznrxMtWOi1GrYjFMeCwDA2YQgjwpv2HXtFBFuFBNVSff2aRbSa0dFnLp7zRijGY9cpPWJ3lH+gyWE2ocubqEHLmzuC80RZZhWKzzMqE/LeL9tlcLDNOH/euvXbYfVv2N9LdqRrG9WJOiSVrVVuVKYbjq/kSIL1Vsn1rvM9fM3tg8Y5IsKM0Y1qkbq0z/00EOfLVNWrse37/wmNdWhYXV9OG+HjqXnaNi1bXX/hc00ePwKzVh3oNhDzdtf6q95Ww+pUc0qapj//EbPZnFaXMYV/wob0LmBJq9KPO3XBfL1H3vr1vcWBu18AAD8FvTIo8KLi47UK7eep2evb+cXVO3yzl1dfJ+/fsf5pR5vjFGHhtUVGRGmxnFVdUf3xoqMCNOwa9v6jik68v1btagdo0G9m6pWTJSu7VRfn9zbXff0aarbuzcJ+LOpFB6mCQ/1Uq/mccV6+CXvMtxt6sVq9M0dfXctLmlVW0ueuUL//l0X9Woepzfu6KzIiDBFR0Xol79dqql/vVD3X9hMxhi9e3dXrf9HP/30eF/fOatGhis8zPvsQss6J9dBGHVzR99djktb1z7l9zq4bwud36SG7ul9jl67vbPG3df9NH9aXm/e6f93+N7dXdStaVyAo6X37u6qy9vUOa1rOPlcQ/emgdegAACUb0w/CQSZZVlauP2IqkZFqHPj02sHKpCd6wnJm47TlZaVqxvfWaCk45n64Pfd1Kt58XUBzsTy3Uc1fe1+3dq1kdoGCLd7jqTrWEa2OjWqodmbDuoP45YVO+baTvX93lAVyMnz6IslezRq+ibd2rWRMnLyZCTFVI4oNqNQ89rRmv7wRapcKVxfLNnz/+3deXxU1d3H8c8vIUBYwia7C4ggIFZQAUEBcW+Fqn26uVur1ceqtdVWrdZqH2utdW3Fpz5SV9RWqqKtijsCUhEFZZEdoiD7TkjIMnOeP+5JmExmkklIZkm+79drXpO595yZO7+5N/d3l3MOt05ZyKCe7Xj5v0eSlWX0uum1irK9D2jN/5w1iN6dW9OzfS7rdxZxwcTZhMKOP587hG8//GGl944eUO3bR/Xg1c/3XTEo7zpta0Ex05Zuplm20Tw7i2MO6cD3Hv0Pe4pDPH3pMByONduKOLZXB469850a4xvt1auPr7hlrbxdRSxXndiHR6atrHidnWVx209cdkJvJkb04zyyTydmrdxa62VLtX5d23Bgh1a8t2RTqvexlBEAABiPSURBVBdFRGqpKXU/qUReRGrFOUdxWbhiQLFUm792B7k52Zz6wPSKaQ/+YDBnD+lZq/eZunA9V06aS0628f4NJ1bpOnVnYSl5uc0qGjdHJvL3f/+oKg21nXM4F1xNuejxj5m+bDMAg3rmMaZfZya8vy8xfv7y4zj3sY8A+NaR3Xjk/Pj7gnDYEXKuSqPvGcs38/Lcrxl/VA9uemk+2wtLufOsQdz1xmJ2FJZWeZ8Zvxpb0R4CYMWm3Zxy//Qq5Qb1zOOe/zqKb/15BgB9u7ThyUuHcc6ED9m0u7hK+fy7zyQUdryxcD3ZZpx+RLcaG0SfPbgHV4zpQ/9ubTEzDr/1jUq3ZSUqMo77K//uM9laUMwxdThASkenDOjCO4t1UCJNQ1NK5HWPvIjUipmlTRIPxGwE7aj9CYozBnXnnV+MoV1uDp3btqgyv13U6MNXjD6UR6ev4gB/m1I0M6u4DejpS4fx7/nr+GLdLs4ddjCd2jRn6sINbNpVzF8vPIYRfTrx+3MGkb9lD1eMiT+qMQQHBllUvcVpVN/OjOob3G4088aTKCwO0a5VDucc3ZPSUJhWzZvhnGP26m0c1LFVRbuDcn06t6F/t7Ys2bC70vShvToysEcef/jOkXz65XauHnsYPdvn8vEtp1BUEuJvM1dx71vLACq6Vc3OMsZ9o0fFe7z989H8fc4aTj+iGy/P+7pSm4sbTuvH+cMPqegOFuAfV4zgqkmfsm7n3pgNs8stvON0fv3SAj5csYUfj+rNiD6dqlztSNQt3xrAo9NXsqWgpOK2rVCME12j+h7AE5cMZfryzRVXg3p1akX+1sJq339g9zy+WL+r0rRY9YYc3J55Ecs//qge7CgsYcbyLbX+TuX6dmnDxIuHsmn3Xob9/t06v0+kBbefxpG3v1Wnukf0yGPRul01F0yCnu1z+XpHUc0FJWOM7lf9bZeNjc7Ii0ijEHmG/K8XHMMZg+KPJVAfwuEgKR7QvS3tW1Udf6Am6XZlA6CwpIzP1+wkL7cZVz83j/atcnjmx8Np06L6cz4vzFnDx/nbuOrEPhzaOfYI0OX+9fk6rnl+HgAHtGnBJ7eeErNc5KBxN/5zPv/4ZE2VMrHOum3atZcX537NA+8sIxR2PH/5cRx1UDsWrN3J3K+2xxwB+qIRh3DLmQMoDTnmrN7GiD6daJmTzcZdexl+177E9/pT+3H+cYdUGW/COce8NTtYv2MvP31uLl3atuDxS4ayZMNubpgcdCf772tO4K7XF1fcZvST0Ydy0xn9ccAn+dvo3z2Pdrk5Fe83f+1OWrdoxmFd2lAaCvP5mh0sWreL3766qNJnnzOkJ78dP5At/jasCe+vYHvUFZiJFx3LKQO7AvDfkz7ljYUbqsTg0QuP4X+nreSzNcFBRJ/OrRn3jR4xu6wtH+hua0Ex50+cTduWzZhw3tEM87GKlxwf1DGX6b8cC8CSDbu56cX5fL52Z5Vy0Z67fDgbdu7lvSWbuOG0w3nriw11Gsk7lvy7z+TWKQuY9FHVBv39urbh8G55/OvzxBvKf3bbqZV67bp9/EBym2fzzEdfsvDr/T94qe7AdnS/znz7qB4V69z+ys3JZt5tp9L/N1NrLpxGXrpqJEcfnNy2P7q1pp4pkRdpeu5/exl/fnc5vTq14t3rT6xoeCt1Ez36cn2+761TFrJs425+d9aguG0hIu0sKuXVz9fRoVUOVz83r2J6dZfPw2FHSajqgdLM5Vt4bcF6+nZpQ3aW8c1B3eiS1zLme5SUhSu6PG3RLKtiLImavl8o7GiWnUUo7Hh/ySbat8rh2F4d+XpHEde/8BltW+bw0A8Hxx3TIp5Q2HHJEx+zZMNuHvrBYEb06RTzN3roneU88E5wlWTwQe15ybfrAHhz0QaueKbqvvHJHw2lV6fWnP3Ih4TDjslXjuTwbm0r2uss27ibcX+ZSTjseOHKETETJeccu/aW0S43p9KBNVAxpkXkLWvrdxbxymfrGNOvM53btojbziP6d/5i3a6KW71q4/QjunLf9wdzwh/fY0dhKXeePYgLjjuEW15ewLOzKyfyT1wylLG+0fr2PSW0b5XDys0FXPLEHNZu33eQ8tHNJ9O+VQ5PzcrnFD/WSeR3//tPjqtoS7SnuIzWUQfFZaEwO4pK6diqOd/531l8sW4XJaHYt5U9fN4Qxn2jB5c//Qlvf7Gx0ry2LZsx/Zdj6dC6OWWhMIfdErur3um/HMsf31zCa/PXAzDpx8O54G+zY5a9fFRvbjlzINv3lHDFM58y58ttJJIynnFEN649uS9tWzZj3podzF61lfeXbMLMqr36MeG8o/npc3MBeOyiY7n5pflsKSip+QOjTL1uFP27JbcDASXy9UyJvEjT45xj6cbd9OrUOq3Ockv9yd+yhxPvnbbvdRLug52Tv43X5q/n+8cexMAe6TFqcjjsauzJqrCkjOLSMO1b5VRK9p1zPPzeCr7aVkhBcVlF16/TbhhLdpaxtzSEc/tGpY60bU8JZeFwRde01YlMZru3a8mMX42ttutc5xynPTCd5ZsKqsyL9TtPnLGKe95cSklZmLyWzfjXNSfQNa8lxWVhpi5cz40vLgCChtq79pbSs30rrhxzKGbG7r2lbNi5l75+YL8/v7uc+99eVvHez142nJFxDpIguOpTXBame7uWMb/Tv+ev4zdTFjKmX+eEei6LjEFBcRkbd+3lvMdmk5OdxeQrR9As29hRWFoxECHsa9MTL0bj/zKTBV9XvdqRf/eZbNtTwqSPvmRA9zxOHdiV6cs2M/er7Zw3/GAmf7KWP725lObNsvjo5pOrXH1asHYn4x+eCcC93zuqytn/Hu1aMu2XY+N21hCvUX351bk12wopCYXp07kNT/8nn9teqXwF6pHzj+aGyZ9TWBKK+f4dWzfn01tPaZCTENVRIl/PlMiLiDQ+e0tDFZf5WzfPZtHvzkjxEmW2cNgx96vt9O+eV+PtU7X11qIN/MSf+X/5qpEMSeBWh3U7ipi2dDNTF22oaBwO1R+wbSkopnXzZlUOPEpDYZwjod6/ikpCnHzfNDbuLubBHwxm/FE9aqxTk0QOtqpTGgqTZRb3ymJZKMxJ933AV9sKuXrsYdxw+uGV5q/esodrnp9Lu9wcvnFge6Yu3MB1p/TlrMHVdwIQCjtmLN9Mn85tKjWGjzQnfxs7Cks5qX8X+kQ0ZL9t3EDOG35wtSdS5n21nXMemVVlete8Fsz+deXb7EpDYR79YCVPzspnS0EJZw3uwUM/HMLe0hChsGNLQTF3vraYgzq04obT+7Fo3S6O7NkuJSdylMjXMyXyIiKN0+xVW3l9wXp+MPTgtDlDLlU55/j0y+20zMlmUM/ER8MGmLViC+dNDG73GN2vM09fOqwhFrGS0lCYnUWlHNCmakP3dFVUEmLVlgIGds9L+hnocq/NX89NL85n+KGdeOyiYxJajuc//oqlG3ZXOjM/sHser/9sVNw6paFwlZ660okS+XqmRF5ERCRzPfHhalZuLuCak/rSNU4bBkkPZaFwQiOOR/tg2WYufvxjsgymXje60q1DmUbdT4qIiIh4Pzq+d6oXQRJUlyQeghHA3/nFGHKbZ1fpDlcSp0ReRERERJLusC7Vd1crNUvfG45ERERERCQuJfIiIiIiIhlIibyIiIiISAZSIi8iIiIikoGUyIuIiIiIZCAl8iIiIiIiGUiJvIiIiIhIBlIiLyIiIiKSgZTIi4iIiIhkICXyIiIiIiIZSIm8iIiIiEgGUiIvIiIiIpKBlMiLiIiIiGQgJfIiIiIiIhlIibyIiIiISAZSIi8iIiIikoHMOZfqZah3ZrY1Nze344ABA1K9KCIiIiLSiC1evJiioqJtzrlOyf7sxprIrwbygPwkf3R//7wkyZ+b6RS3ulHc6kZxqxvFrW4Ut7pR3OpGcaub/Y1bL2CXc653/SxO4hplIp8qZvYpgHPumFQvSyZR3OpGcasbxa1uFLe6UdzqRnGrG8WtbjI5brpHXkREREQkAymRFxERERHJQErkRUREREQykBJ5EREREZEMpEReRERERCQDqdcaEREREZEMpDPyIiIiIiIZSIm8iIiIiEgGUiIvIiIiIpKBlMiLiIiIiGQgJfIiIiIiIhlIibyIiIiISAZSIi8iIiIikoGUyNcDMzvQzB43s3VmVmxm+Wb2oJl1SPWy1Rcz62Rml5nZy2a2wsyKzGynmc00sx+bWcx1ycxGmtnrZrbNzArNbL6ZXWdm2dV81jgzm+bfv8DMZpvZxTUs38Vm9rEvv9PXH7e/37uhmNmFZub847I4ZRo8DmaW7X+P+f433eZ/r5H7+x3ri5mNMrMXzWy9377Wm9lbZvatGGW1vgFmdqaP0Vr/u64ys8lmNiJO+SYRNzP7rpn9xcxmmNkuv/1NqqFOWsYmmdtubeJmZn3N7EYze8/M1phZiZltNLNXzGxsDZ/T4DEws1wzu8PMlprZXjPbZGYvmNmAxCOSmLqsb1H1/2b79hOHxSmTlBiYWUcL8pp8C/4Pr7Mg7zkw0e+TqDpup+bXn2k+BkVmttp/r35x6jSO9c05p8d+PIA+wEbAAVOAu4H3/OslQKdUL2M9fc8r/XdaBzwL/AF4HNjhp/8TP8BYRJ2zgDKgAPgb8CcfEwdMjvM5V/v5W4AJwAPAGj/t3jh17vXz1/jyE4CtftrVqY5djOU9yMdtt1/Gy1IRB8CAyRHr6p/871Tgf7ez0iBWt/rl2ww8AdwF/B8wB7hH61vM5ftjxHea6P8n/RMoAcLABU01bsBn/vN2A4v935OqKZ+WsUn2tlubuAF/9/MXAY8S7Cte8svlgGtTFQOgBTDT15njt5XngFJgDzA8letbVN3xEXUdcFiqYgB0Apb6Ou8S/E+Z4l9vBA5N8XbaEvhXRBwe9uvdU8AqYFxjXt/qLfBN9QG86X+ka6Km3++n/zXVy1hP3/Mk/48lK2p6N+Ar/13/K2J6HrAJKAaOjZjeEpjly/8w6r16AXv9xtQrYnoHYIWvMyKqzkg/fQXQIeq9tvr367U/372e42jAO8BK/4+gSiKfrDgA5/o6HwItI6YP9b/bJqBtCmP1Pb98b8daDiBH61uVmHQDQsAGoEvUvLF+2Vc11bj5GPT12+GJVJ+Qpm1sSPK2W8u4XQIMiTF9DMHBZDHQPRUxAG72dSYTsS8jOGArP/jIqikeDRG3qHqdCbbhvwPTiJ/IJyUGBAdkDrg/avq1fvrUVG2nvvwEX+auWL8fEfuKxri+1Vvgm+IDONT/GKtjrPhtCY7U9gCtU72sDRyHX/s4/CVi2qV+2lMxyp/k530QNf13fvodMerEfD/gaT/9RzHqxH2/FMbqZwRnRUcDtxM7kU9KHIDpfvrYGHXivl+S4pRFcCZlD9A5gfJa34JlGO6X4ZU483cBuxU3BzUnpGkbm1RuuzXFrYa6bxF10idZMSBICr/003vHqBP3/ZIdN+BlgkS+E9Un8g0eA6A1UEiQz0QnqlkE+Y+jns/KJxo3grsiQsDHRN0VUM17Nqr1TffI75+T/PNbzrlw5Azn3G6CI7dWwHHJXrAkK/XPZRHTymMzNUb56QT/GEaaWYsE67wRVWZ/6qSEvyfubuAh59z0aoo2eBx83EcS/A4zavE5yTIS6A28Dmy34J7vG83sZxb7Pm+tb4HlBGc9h5nZAZEzzGw0wQmGdyImK27xpWVsMmDbrU6sfQUkJwZ9gIOBZc651QnWSTozuwQ4G7jSObe1mnLJisEIIBf40Oc1FXze85Z/WW37hwZ0LsEBxVNAnpldYGY3m9lP4rUroJGtb0rk98/h/nlZnPnL/XPMhhaNgZk1Ay7yLyM3irixcc6VERzFNyO4qpFInfUEZ2cPNLNW/rNbAz2BAj8/WtrE38fpGYLbkH5dQ/FkxOEwIJvgNovonWq8Osk01D9vBOYC/yY4CHoQmGVmH5hZ54jyWt8A59w24EagK/CFmf2fmf3BzF4g2OG+DVwRUUVxiy9dY5Pu225MZnYIcDJBMjQ9YnqyYpD2+2sfo4cIzj5PqaF4smKQ7nEr31e0I7hl9RmCW2weBZaZ2QSLaJjeGNc3JfL7p51/3hlnfvn09klYllS5GxgEvO6cezNiel1ik2iddlHPmRD/24AhwCXOuaIayiYjDukeuy7++UqCs0GnEJxNHkTQLmU0wX2H5bS+ec65B4HvECSZlwM3EbQ3WAM86ZzbFFFccYsvXWOTcfH0ZzSfJWj8d7tzbnvE7GTFIK3jZkHPb08R3MJybQJVFLdA+b7id8AnwJEE+4qTCRL7q4DfRJRvdHFTIt+wzD+7lC5FAzGza4HrCVpwX1jb6v65NrGpazxTGn8zG0ZwFv4+59x/6uMt/XNDxiHV6275GRQDvuuce9c5V+CcWwScA6wFxsS5zSaWprS+/Yqgl5onCS7vtgaOIWhz8KyZ3VObt/PPjT5udZCusUn1tluJPxv6DHA88A+C3kLqoqFjkOq4/ZygQfDlUQc6dZWsGKQ6buX7ivXAOc65hX5f8R7wXYI2ab8ws+a1fN+MiZsS+f0TfXYlWl5UuUbDzH5KcAnwC4LGGtuiitQlNonW2ZVg+ZqOiBtcxC01y6h8VqA6yYhDuq+75TuyVc65zyNn+Csa5Vd/hvlnrW+AmZ1I0MXZq865XzjnVjnnCp1zcwkOgL4Grjez8ttBFLf40jU26b7tVvBJ/CSCK0IvEHR9Gp24JCsGaRs3M+sL/B54wjn3eoLVkhWDtI2bV76vmBp9tdvvO1YTnKEv77e90a1vSuT3z1L/HO8ep77+Od49UhnJzK4j6Kd1IUESvyFGsbix8cltb4IGT6sSrNOd4MziWudcIYBzbg9BYtLGz4+WDvFvQ/B9BgB7Iwb3cMBvfZnH/LQH/etkxGEFQUv/Q/3vkUidZCqPwY4488v/eedGlW/q61v5YCbvR8/w3+Njgv/7Q/xkxS2+dI1Num+7QEWMngd+SNB39nmx7i9OYgzSeX99BMFtRz+K3Ef4/cQYX2a5n3a2f52sGKRz3KCW+4rGuL4pkd8/5TvL0yxqZFMza0twKbEI+CjZC9ZQzOxGgsETPiNI4jfFKfqefz4jxrzRBL35zHLOFSdY55tRZfanTjIVEwwaEesxz5eZ6V+X33bT4HHwcZ9F8DuMqsXnJMt0giSpb5xLooP8c75/1voWKO9BpXOc+eXTS/yz4hZfWsYmA7Zd/Db7T4Iz8U8DFzrnQtVUSUYMVhJ0NtDPzHonWCdZ8om/nyg/UTbZv86HpMbgI4I85nif11Twec9p/mWVkwdJ8q5/HhQ9w7fNKE+Y8yNmNa71bX/7r2zqD5rIgFD+O/3Gf6dPgI41lM0jGI2zNoOp9CZDB5qpYzxvJ3Y/8kmJA4kNcJGXwvhM8st3Z9T0Uwnue9wBtNf6Vmn5vu+XbwPQM2reN33civAjTjfluJHYgFBpGZtUbrsJxK0F8JovM5EEBrxJVgxI8oBQtYlbNfWmEb8f+aTEgH0DQt0XNb1BBoSq5frWnCBpDgOnRs2709ed1pjXtwYJfFN6EDQm2+h/lCkEwwK/518vxe8wM/0BXOy/UxnBGfnbYzwuiapzNvuGN58I3EPE8ObEGLwBuMbPr83w5vf5+ZFDLW/x05Iy9HsdY3o7MRL5ZMWBykNOL/a/T4MN816H+HQh6KLLEZyhv9cvbxlBf9Tf0/pWZdmyCLqYdAT3YT+Fv2eeYEfngJ811bj57/qkf0z1n70yYtq9McqnXWxI8rZbm7gBT/j5m4E7iL2vODEVMSA4yPjQ15lD0OvacwT/T/YAw1O5vsV5j2nET+STEgOCgamW+jrvEuQ5U/zrjUCfFG+nJxB0a1rm43Ev8IGvtwno15jXt3oLfFN+AAcR/PNaT3DJ+kuChqDVnrXOpAf7ks7qHtNi1DseP6gPwZnABQSt87Or+azxfiPc7Vf2OcDFNSzfxb7cHl/vA2BcquOWYEyrJPLJigNBF4U/979Lkf+dXgdGpjo+fvk6ElzdWu23ra3AK8Bxcco3+fUNyAGuI7gkvsvvZDYR9MV/WlOOWwL/x/IzJTbJ3HZrEzf2JZ7VPW5PVQwI7pW+g+AkQTHBAcdkYGA6rG8x3qM8nlUS+WTGgOB/8UME+U0JQb7zOHBgOsQNGEjQK9Imv3xrCK4kxF2+xrK+mf8gERERERHJIGrsKiIiIiKSgZTIi4iIiIhkICXyIiIiIiIZSIm8iIiIiEgGUiIvIiIiIpKBlMiLiIiIiGQgJfIiIiIiIhlIibyIiIiISAZSIi8iIiIikoGUyIuIiIiIZCAl8iIiIiIiGUiJvIiIiIhIBlIiLyIiIiKSgZTIi4iIiIhkICXyIiIiIiIZSIm8iIiIiEgGUiIvIiIiIpKB/h/+uiitHug8hgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 377
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses['train'], label='Training loss')\n",
    "plt.legend()\n",
    "_ = plt.ylim()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取 Tensors\n",
    "使用函数 get_tensor_by_name()从 loaded_graph 中获取tensors，后面的推荐功能要用到。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tensors(loaded_graph):\n",
    "\n",
    "    uid = loaded_graph.get_tensor_by_name(\"uid:0\")\n",
    "    user_gender = loaded_graph.get_tensor_by_name(\"user_gender:0\")\n",
    "    user_age = loaded_graph.get_tensor_by_name(\"user_age:0\")\n",
    "    user_job = loaded_graph.get_tensor_by_name(\"user_job:0\")\n",
    "    movie_id = loaded_graph.get_tensor_by_name(\"movie_id:0\")\n",
    "    movie_categories = loaded_graph.get_tensor_by_name(\"movie_categories:0\")\n",
    "    movie_titles = loaded_graph.get_tensor_by_name(\"movie_titles:0\")\n",
    "    targets = loaded_graph.get_tensor_by_name(\"targets:0\")\n",
    "    dropout_keep_prob = loaded_graph.get_tensor_by_name(\"dropout_keep_prob:0\")\n",
    "    lr = loaded_graph.get_tensor_by_name(\"LearningRate:0\")\n",
    "    #两种不同计算预测评分的方案使用不同的name获取tensor inference\n",
    "#     inference = loaded_graph.get_tensor_by_name(\"inference/inference/BiasAdd:0\")\n",
    "    inference = loaded_graph.get_tensor_by_name(\"inference/ExpandDims:0\") # 之前是MatMul:0 因为inference代码修改了 \n",
    "    movie_combine_layer_flat = loaded_graph.get_tensor_by_name(\"movie_fc/Reshape:0\")\n",
    "    user_combine_layer_flat = loaded_graph.get_tensor_by_name(\"user_fc/Reshape:0\")\n",
    "    return uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, inference, movie_combine_layer_flat, user_combine_layer_flat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 根据用户ID和电影ID，预测评分，就是网络的正向传播"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rating_movie(user_id_val, movie_id_val):\n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "    \n",
    "        # Get Tensors from loaded model\n",
    "        uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, inference,_, __ = get_tensors(loaded_graph)  #loaded_graph\n",
    "    \n",
    "        categories = np.zeros([1, 18])\n",
    "        categories[0] = movies.values[movieid2idx[movie_id_val]][2]\n",
    "    \n",
    "        titles = np.zeros([1, sentences_size])\n",
    "        titles[0] = movies.values[movieid2idx[movie_id_val]][1]\n",
    "    \n",
    "        feed = {\n",
    "              uid: np.reshape(users.values[user_id_val-1][0], [1, 1]),\n",
    "              user_gender: np.reshape(users.values[user_id_val-1][1], [1, 1]),\n",
    "              user_age: np.reshape(users.values[user_id_val-1][2], [1, 1]),\n",
    "              user_job: np.reshape(users.values[user_id_val-1][3], [1, 1]),\n",
    "              movie_id: np.reshape(movies.values[movieid2idx[movie_id_val]][0], [1, 1]),\n",
    "              movie_categories: categories,  #x.take(6,1)\n",
    "              movie_titles: titles,  #x.take(5,1)\n",
    "              dropout_keep_prob: 1}\n",
    "    \n",
    "        # Get Prediction\n",
    "        inference_val = sess.run([inference], feed)  \n",
    "    \n",
    "        return (inference_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([[3.9628615]], dtype=float32)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_movie(234, 1401)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 生成Movie特征矩阵\n",
    "将训练好的电影特征组合成电影特征矩阵并保存在本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n"
     ]
    }
   ],
   "source": [
    "loaded_graph = tf.Graph()  #\n",
    "movie_matrics = []\n",
    "with tf.Session(graph=loaded_graph) as sess:  #\n",
    "    # Load saved model\n",
    "    loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "    loader.restore(sess, load_dir)\n",
    "\n",
    "    # Get Tensors from loaded model\n",
    "    uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, _, movie_combine_layer_flat, __ = get_tensors(loaded_graph)  #loaded_graph\n",
    "\n",
    "    for item in movies.values:\n",
    "        categories = np.zeros([1, 18])\n",
    "        categories[0] = item.take(2)\n",
    "\n",
    "        titles = np.zeros([1, sentences_size])\n",
    "        titles[0] = item.take(1)\n",
    "\n",
    "        feed = {\n",
    "            movie_id: np.reshape(item.take(0), [1, 1]),\n",
    "            movie_categories: categories,  #x.take(6,1)\n",
    "            movie_titles: titles,  #x.take(5,1)\n",
    "            dropout_keep_prob: 1}\n",
    "\n",
    "        movie_combine_layer_flat_val = sess.run([movie_combine_layer_flat], feed)  \n",
    "        movie_matrics.append(movie_combine_layer_flat_val)\n",
    "\n",
    "pickle.dump((np.array(movie_matrics).reshape(-1, 200)), open('movie_matrics.p', 'wb'))\n",
    "movie_matrics = pickle.load(open('movie_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_matrics = pickle.load(open('movie_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 生成 User特征矩阵\n",
    "将训练好的用户特征组合成用户特征矩阵并保存在本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n"
     ]
    }
   ],
   "source": [
    "loaded_graph = tf.Graph()  #\n",
    "users_matrics = []\n",
    "with tf.Session(graph=loaded_graph) as sess:  #\n",
    "    # Load saved model\n",
    "    loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "    loader.restore(sess, load_dir)\n",
    "\n",
    "    # Get Tensors from loaded model\n",
    "    uid, user_gender, user_age, user_job, movie_id, movie_categories, movie_titles, targets, lr, dropout_keep_prob, _, __,user_combine_layer_flat = get_tensors(loaded_graph)  #loaded_graph\n",
    "\n",
    "    for item in users.values:\n",
    "\n",
    "        feed = {\n",
    "            uid: np.reshape(item.take(0), [1, 1]),\n",
    "            user_gender: np.reshape(item.take(1), [1, 1]),\n",
    "            user_age: np.reshape(item.take(2), [1, 1]),\n",
    "            user_job: np.reshape(item.take(3), [1, 1]),\n",
    "            dropout_keep_prob: 1}\n",
    "\n",
    "        user_combine_layer_flat_val = sess.run([user_combine_layer_flat], feed)  \n",
    "        users_matrics.append(user_combine_layer_flat_val)\n",
    "\n",
    "pickle.dump((np.array(users_matrics).reshape(-1, 200)), open('users_matrics.p', 'wb'))\n",
    "users_matrics = pickle.load(open('users_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_matrics = pickle.load(open('users_matrics.p', mode='rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 开始推荐电影\n",
    "使用生产的用户特征矩阵和电影特征矩阵做电影推荐"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 推荐同类型的电影\n",
    "思路是计算当前看的电影特征向量与整个电影特征矩阵的余弦相似度，取相似度最大的top_k个，这里加了些随机选择在里面，保证每次的推荐稍稍有些不同。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_same_type_movie(movie_id_val, top_k = 20):\n",
    "    \n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "        \n",
    "        norm_movie_matrics = tf.sqrt(tf.reduce_sum(tf.square(movie_matrics), 1, keep_dims=True))\n",
    "        normalized_movie_matrics = movie_matrics / norm_movie_matrics\n",
    "\n",
    "        #推荐同类型的电影\n",
    "        probs_embeddings = (movie_matrics[movieid2idx[movie_id_val]]).reshape([1, 200])\n",
    "        probs_similarity = tf.matmul(probs_embeddings, tf.transpose(normalized_movie_matrics))\n",
    "        sim = (probs_similarity.eval())\n",
    "    #     results = (-sim[0]).argsort()[0:top_k]\n",
    "    #     print(results)\n",
    "        \n",
    "        print(\"您看的电影是：{}\".format(movies_orig[movieid2idx[movie_id_val]]))\n",
    "        print(\"以下是给您的推荐：\")\n",
    "        p = np.squeeze(sim)\n",
    "        p[np.argsort(p)[:-top_k]] = 0\n",
    "        p = p / np.sum(p)\n",
    "        results = set()\n",
    "        while len(results) != 5:\n",
    "            c = np.random.choice(3883, 1, p=p)[0]\n",
    "            results.add(c)\n",
    "        for val in (results):\n",
    "            print(val)\n",
    "            print(movies_orig[val])\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n",
      "您看的电影是：[1401 'Ghosts of Mississippi (1996)' 'Drama']\n",
      "以下是给您的推荐：\n",
      "941\n",
      "[953 \"It's a Wonderful Life (1946)\" 'Drama']\n",
      "1168\n",
      "[1185 'My Left Foot (1989)' 'Drama']\n",
      "3413\n",
      "[3482 'Price of Glory (2000)' 'Drama']\n",
      "568\n",
      "[572 'Foreign Student (1994)' 'Drama']\n",
      "1115\n",
      "[1131 'Jean de Florette (1986)' 'Drama']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{568, 941, 1115, 1168, 3413}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend_same_type_movie(1401, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 推荐您喜欢的电影\n",
    "思路是使用用户特征向量与电影特征矩阵计算所有电影的评分，取评分最高的top_k个，同样加了些随机选择部分。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_your_favorite_movie(user_id_val, top_k = 10):\n",
    "\n",
    "    loaded_graph = tf.Graph()  #\n",
    "    with tf.Session(graph=loaded_graph) as sess:  #\n",
    "        # Load saved model\n",
    "        loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "        loader.restore(sess, load_dir)\n",
    "\n",
    "        #推荐您喜欢的电影\n",
    "        probs_embeddings = (users_matrics[user_id_val-1]).reshape([1, 200])\n",
    "\n",
    "        probs_similarity = tf.matmul(probs_embeddings, tf.transpose(movie_matrics))\n",
    "        sim = (probs_similarity.eval())\n",
    "    #     print(sim.shape)\n",
    "    #     results = (-sim[0]).argsort()[0:top_k]\n",
    "    #     print(results)\n",
    "        \n",
    "    #     sim_norm = probs_norm_similarity.eval()\n",
    "    #     print((-sim_norm[0]).argsort()[0:top_k])\n",
    "    \n",
    "        print(\"以下是给您的推荐：\")\n",
    "        p = np.squeeze(sim)\n",
    "        p[np.argsort(p)[:-top_k]] = 0\n",
    "        p = p / np.sum(p)\n",
    "        results = set()\n",
    "        while len(results) != 5:\n",
    "            c = np.random.choice(3883, 1, p=p)[0]\n",
    "            results.add(c)\n",
    "        for val in (results):\n",
    "            print(val)\n",
    "            print(movies_orig[val])\n",
    "\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./save\n",
      "以下是给您的推荐：\n",
      "3366\n",
      "[3435 'Double Indemnity (1944)' 'Crime|Film-Noir']\n",
      "3815\n",
      "[3885 'Love & Sex (2000)' 'Comedy|Romance']\n",
      "910\n",
      "[922 'Sunset Blvd. (a.k.a. Sunset Boulevard) (1950)' 'Film-Noir']\n",
      "847\n",
      "[858 'Godfather, The (1972)' 'Action|Crime|Drama']\n",
      "1242\n",
      "[1262 'Great Escape, The (1963)' 'Adventure|War']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{847, 910, 1242, 3366, 3815}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend_your_favorite_movie(234, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
